backup: keep every KVM export checkpoint and start from the one the client names

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -134,12 +134,6 @@ public interface VmDetailConstants {
     String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
     String CLOUDSTACK_VLAN = "cloudstack.vlan";
 
-    // KVM Checkpoints related
-    String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id";
-    String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time";
-    String LAST_CHECKPOINT_ID = "last.checkpoint.id";
-    String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time";
-
     // KBOSS specific
     String LINKED_VOLUMES_SECONDARY_STORAGE_UUIDS = "linkedVolumesSecondaryStorageUuids";
     String VALIDATION_COMMAND = "backupValidationCommand";

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java

@@ -66,6 +66,12 @@ public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd {
             description = "the description for the backup")
     private String description;
 
+    @Parameter(name = ApiConstants.FROM_CHECKPOINT_ID,
+            type = CommandType.STRING,
+            description = "ID of the checkpoint the incremental backup starts from. It must be a checkpoint of the VM "
+                    + "known to CloudStack (see listVmCheckpoints). When omitted a full backup is taken.")
+    private String fromCheckpointId;
+
     public Long getVmId() {
         return vmId;
     }
@@ -78,6 +84,10 @@ public String getDescription() {
         return description;
     }
 
+    public String getFromCheckpointId() {
+        return fromCheckpointId;
+    }
+
     @Override
     public void execute() {
         try {

api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java

@@ -50,4 +50,16 @@ public void setCreated(Date created) {
     public void setIsActive(Boolean isActive) {
         this.isActive = isActive;
     }
+
+    public String getId() {
+        return id;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public Boolean getIsActive() {
+        return isActive;
+    }
 }

api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java

@@ -94,9 +94,10 @@ public interface KVMBackupExportService extends Configurable, PluggableService {
      */
     List<CheckpointResponse> listVmCheckpoints(ListVmCheckpointsCmd cmd);
 
-    /**
-     * Delete a VM checkpoint (no-op for normal flow, kept for API parity)
-     */
+    // Checkpoints recorded for the VM, oldest first. The newest is marked active.
+    List<CheckpointResponse> listVmCheckpoints(long vmId);
+
+    // Delete a VM checkpoint: its libvirt checkpoint or dirty bitmaps on the host, then its record.
     boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd);
 
     /**

core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java

@@ -20,21 +20,32 @@
 import java.util.Map;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.LogLevel;
 
 public class DeleteVmCheckpointCommand extends Command {
     private String vmName;
     private String checkpointId;
     private Map<String, String> diskPathUuidMap;
+    @LogLevel(LogLevel.Log4jLevel.Off)
+    private Map<String, byte[]> diskPathPassphraseMap;
     private boolean stoppedVM;
+    private Long checkpointCreateTime;
 
     public DeleteVmCheckpointCommand() {
     }
 
     public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map<String, String> diskPathUuidMap, boolean stoppedVM) {
+        this(vmName, checkpointId, diskPathUuidMap, null, stoppedVM, null);
+    }
+
+    public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map<String, String> diskPathUuidMap,
+                                     Map<String, byte[]> diskPathPassphraseMap, boolean stoppedVM, Long checkpointCreateTime) {
         this.vmName = vmName;
         this.checkpointId = checkpointId;
         this.diskPathUuidMap = diskPathUuidMap;
+        this.diskPathPassphraseMap = diskPathPassphraseMap;
         this.stoppedVM = stoppedVM;
+        this.checkpointCreateTime = checkpointCreateTime;
     }
 
     public String getVmName() {
@@ -45,10 +56,19 @@ public String getCheckpointId() {
         return checkpointId;
     }
 
+    // Creation time of the checkpoint on the host in epoch seconds, needed to redefine it in libvirt.
+    public Long getCheckpointCreateTime() {
+        return checkpointCreateTime;
+    }
+
     public Map<String, String> getDiskPathUuidMap() {
         return diskPathUuidMap;
     }
 
+    public Map<String, byte[]> getDiskPathPassphraseMap() {
+        return diskPathPassphraseMap;
+    }
+
     public boolean isStoppedVM() {
         return stoppedVM;
     }

engine/schema/src/main/java/org/apache/cloudstack/backup/VmCheckpointVO.java

@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.backup;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+import com.cloud.utils.db.GenericDao;
+
+// A checkpoint of the KVM backup export: a libvirt checkpoint of the running VM and a persistent dirty bitmap of the
+// same name in every disk image. It stays until a client deletes it, so several clients can keep their own chains.
+@Entity
+@Table(name = "vm_checkpoint")
+public class VmCheckpointVO {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private long id;
+
+    @Column(name = "vm_id")
+    private long vmId;
+
+    @Column(name = "checkpoint_id")
+    private String checkpointId;
+
+    @Column(name = "create_time")
+    private Long createTime;
+
+    @Column(name = "backup_id")
+    private Long backupId;
+
+    @Column(name = GenericDao.CREATED_COLUMN)
+    private Date created;
+
+    @Column(name = GenericDao.REMOVED_COLUMN)
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date removed;
+
+    public VmCheckpointVO() {
+    }
+
+    public VmCheckpointVO(long vmId, String checkpointId, Long createTime, Long backupId) {
+        this.vmId = vmId;
+        this.checkpointId = checkpointId;
+        this.createTime = createTime;
+        this.backupId = backupId;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public String getCheckpointId() {
+        return checkpointId;
+    }
+
+    // Creation time of the checkpoint on the host in epoch seconds, as libvirt records it.
+    public Long getCreateTime() {
+        return createTime;
+    }
+
+    public Long getBackupId() {
+        return backupId;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public Date getRemoved() {
+        return removed;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("VmCheckpoint {\"id\": %d, \"vmId\": %d, \"checkpointId\": \"%s\"}", id, vmId, checkpointId);
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDao.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.backup.dao;
+
+import java.util.List;
+
+import org.apache.cloudstack.backup.VmCheckpointVO;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface VmCheckpointDao extends GenericDao<VmCheckpointVO, Long> {
+    // Checkpoints of the VM in the order they were recorded, oldest first.
+    List<VmCheckpointVO> listByVmId(long vmId);
+
+    VmCheckpointVO findByVmIdAndCheckpointId(long vmId, String checkpointId);
+}

engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDaoImpl.java

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.backup.dao;
+
+import java.util.List;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.cloudstack.backup.VmCheckpointVO;
+import org.springframework.stereotype.Component;
+
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+
+@Component
+public class VmCheckpointDaoImpl extends GenericDaoBase<VmCheckpointVO, Long> implements VmCheckpointDao {
+    private SearchBuilder<VmCheckpointVO> vmIdSearch;
+    private SearchBuilder<VmCheckpointVO> vmIdCheckpointIdSearch;
+
+    @PostConstruct
+    protected void init() {
+        vmIdSearch = createSearchBuilder();
+        vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+        vmIdSearch.done();
+
+        vmIdCheckpointIdSearch = createSearchBuilder();
+        vmIdCheckpointIdSearch.and("vmId", vmIdCheckpointIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+        vmIdCheckpointIdSearch.and("checkpointId", vmIdCheckpointIdSearch.entity().getCheckpointId(), SearchCriteria.Op.EQ);
+        vmIdCheckpointIdSearch.done();
+    }
+
+    @Override
+    public List<VmCheckpointVO> listByVmId(long vmId) {
+        SearchCriteria<VmCheckpointVO> sc = vmIdSearch.create();
+        sc.setParameters("vmId", vmId);
+        // Rows are added in finalize order on the management server; the host's creation time is not
+        // comparable across hosts with different clocks.
+        return listBy(sc, new Filter(VmCheckpointVO.class, "id", true, null, null));
+    }
+
+    @Override
+    public VmCheckpointVO findByVmIdAndCheckpointId(long vmId, String checkpointId) {
+        SearchCriteria<VmCheckpointVO> sc = vmIdCheckpointIdSearch.create();
+        sc.setParameters("vmId", vmId);
+        sc.setParameters("checkpointId", checkpointId);
+        return findOneBy(sc);
+    }
+}

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -285,6 +285,7 @@
   <bean id="backupRepositoryDaoImpl" class="org.apache.cloudstack.backup.dao.BackupRepositoryDaoImpl" />
   <bean id="backupUsageMetricDaoImpl" class="org.apache.cloudstack.backup.dao.BackupUsageMetricDaoImpl" />
   <bean id="imageTransferDaoImpl" class="org.apache.cloudstack.backup.dao.ImageTransferDaoImpl" />
+  <bean id="vmCheckpointDaoImpl" class="org.apache.cloudstack.backup.dao.VmCheckpointDaoImpl" />
   <bean id="directDownloadCertificateDaoImpl" class="org.apache.cloudstack.direct.download.DirectDownloadCertificateDaoImpl" />
   <bean id="directDownloadCertificateHostMapDaoImpl" class="org.apache.cloudstack.direct.download.DirectDownloadCertificateHostMapDaoImpl" />
   <bean id="routerHealthCheckResultsDaoImpl" class="com.cloud.network.dao.RouterHealthCheckResultDaoImpl" />

engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql

@@ -241,3 +241,42 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.disk_offering', 'error_policy', 'VARCHAR(16) DEFAULT ''report'' COMMENT ''disk error policy (report/stop/ignore/enospace) used by KVM hypervisor''');
+
+-- VM checkpoints
+CREATE TABLE IF NOT EXISTS `cloud`.`vm_checkpoint`(
+    `id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
+    `vm_id` bigint unsigned NOT NULL COMMENT 'Instance ID',
+    `checkpoint_id` varchar(64) NOT NULL COMMENT 'Checkpoint name, used for the libvirt checkpoint and the disk bitmaps',
+    `create_time` bigint unsigned COMMENT 'Creation time on the host, epoch seconds',
+    `backup_id` bigint unsigned COMMENT 'Backup that created the checkpoint',
+    `created` datetime NOT NULL COMMENT 'date created',
+    `removed` datetime COMMENT 'date removed if not null',
+    PRIMARY KEY (`id`),
+    INDEX `i_vm_checkpoint__vm_id`(`vm_id`),
+    CONSTRAINT `fk_vm_checkpoint__vm_id` FOREIGN KEY (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+-- The previous checkpoint first, so that it gets the lower id.
+INSERT INTO `cloud`.`vm_checkpoint` (`vm_id`, `checkpoint_id`, `create_time`, `created`)
+SELECT i.`vm_id`, i.`value`, CAST(t.`value` AS UNSIGNED), NOW()
+FROM `cloud`.`vm_instance_details` i
+         JOIN `cloud`.`vm_instance` v ON v.`id` = i.`vm_id` AND v.`removed` IS NULL
+         LEFT JOIN `cloud`.`vm_instance_details` t ON t.`vm_id` = i.`vm_id` AND t.`name` = 'last.checkpoint.create.time'
+WHERE i.`name` = 'last.checkpoint.id'
+  AND NOT EXISTS (SELECT 1 FROM `cloud`.`vm_checkpoint` c
+                  WHERE c.`vm_id` = i.`vm_id` AND c.`checkpoint_id` = i.`value` AND c.`removed` IS NULL);
+
+-- The active checkpoint. Skipped when it has the same id as the previous one.
+INSERT INTO `cloud`.`vm_checkpoint` (`vm_id`, `checkpoint_id`, `create_time`, `created`)
+SELECT i.`vm_id`, i.`value`, CAST(t.`value` AS UNSIGNED), NOW()
+FROM `cloud`.`vm_instance_details` i
+         JOIN `cloud`.`vm_instance` v ON v.`id` = i.`vm_id` AND v.`removed` IS NULL
+         LEFT JOIN `cloud`.`vm_instance_details` t ON t.`vm_id` = i.`vm_id` AND t.`name` = 'active.checkpoint.create.time'
+WHERE i.`name` = 'active.checkpoint.id'
+  AND NOT EXISTS (SELECT 1 FROM `cloud`.`vm_checkpoint` c
+                  WHERE c.`vm_id` = i.`vm_id` AND c.`checkpoint_id` = i.`value` AND c.`removed` IS NULL);
+
+-- The details are not read any more.
+DELETE FROM `cloud`.`vm_instance_details`
+WHERE `name` IN ('active.checkpoint.id', 'active.checkpoint.create.time',
+                 'last.checkpoint.id', 'last.checkpoint.create.time');