diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 521633a0c597..cb32b62be385 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -134,12 +134,6 @@ public interface VmDetailConstants { String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details"; String CLOUDSTACK_VLAN = "cloudstack.vlan"; - // KVM Checkpoints related - String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id"; - String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time"; - String LAST_CHECKPOINT_ID = "last.checkpoint.id"; - String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time"; - // KBOSS specific String LINKED_VOLUMES_SECONDARY_STORAGE_UUIDS = "linkedVolumesSecondaryStorageUuids"; String VALIDATION_COMMAND = "backupValidationCommand"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java index 1bf6d45db049..2c1af8627ec9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/StartBackupCmd.java @@ -66,6 +66,12 @@ public class StartBackupCmd extends BaseAsyncCreateCmd implements AdminCmd { description = "the description for the backup") private String description; + @Parameter(name = ApiConstants.FROM_CHECKPOINT_ID, + type = CommandType.STRING, + description = "ID of the checkpoint the incremental backup starts from. It must be a checkpoint of the VM " + + "known to CloudStack (see listVmCheckpoints). When omitted a full backup is taken.") + private String fromCheckpointId; + public Long getVmId() { return vmId; } @@ -78,6 +84,10 @@ public String getDescription() { return description; } + public String getFromCheckpointId() { + return fromCheckpointId; + } + @Override public void execute() { try { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java index 2bec7711064f..b7c5f1d8840a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CheckpointResponse.java @@ -50,4 +50,16 @@ public void setCreated(Date created) { public void setIsActive(Boolean isActive) { this.isActive = isActive; } + + public String getId() { + return id; + } + + public Date getCreated() { + return created; + } + + public Boolean getIsActive() { + return isActive; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java index e1affb311ec6..94e7bfa20513 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/KVMBackupExportService.java @@ -94,9 +94,10 @@ public interface KVMBackupExportService extends Configurable, PluggableService { */ List listVmCheckpoints(ListVmCheckpointsCmd cmd); - /** - * Delete a VM checkpoint (no-op for normal flow, kept for API parity) - */ + // Checkpoints recorded for the VM, oldest first. The newest is marked active. + List listVmCheckpoints(long vmId); + + // Delete a VM checkpoint: its libvirt checkpoint or dirty bitmaps on the host, then its record. boolean deleteVmCheckpoint(DeleteVmCheckpointCmd cmd); /** diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java index 81cf6c1abfcc..03fa2d23f6aa 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteVmCheckpointCommand.java @@ -20,21 +20,32 @@ import java.util.Map; import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; public class DeleteVmCheckpointCommand extends Command { private String vmName; private String checkpointId; private Map diskPathUuidMap; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map diskPathPassphraseMap; private boolean stoppedVM; + private Long checkpointCreateTime; public DeleteVmCheckpointCommand() { } public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, boolean stoppedVM) { + this(vmName, checkpointId, diskPathUuidMap, null, stoppedVM, null); + } + + public DeleteVmCheckpointCommand(String vmName, String checkpointId, Map diskPathUuidMap, + Map diskPathPassphraseMap, boolean stoppedVM, Long checkpointCreateTime) { this.vmName = vmName; this.checkpointId = checkpointId; this.diskPathUuidMap = diskPathUuidMap; + this.diskPathPassphraseMap = diskPathPassphraseMap; this.stoppedVM = stoppedVM; + this.checkpointCreateTime = checkpointCreateTime; } public String getVmName() { @@ -45,10 +56,19 @@ public String getCheckpointId() { return checkpointId; } + // Creation time of the checkpoint on the host in epoch seconds, needed to redefine it in libvirt. + public Long getCheckpointCreateTime() { + return checkpointCreateTime; + } + public Map getDiskPathUuidMap() { return diskPathUuidMap; } + public Map getDiskPathPassphraseMap() { + return diskPathPassphraseMap; + } + public boolean isStoppedVM() { return stoppedVM; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/VmCheckpointVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/VmCheckpointVO.java new file mode 100644 index 000000000000..b7a65543f6e8 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/VmCheckpointVO.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.utils.db.GenericDao; + +// A checkpoint of the KVM backup export: a libvirt checkpoint of the running VM and a persistent dirty bitmap of the +// same name in every disk image. It stays until a client deletes it, so several clients can keep their own chains. +@Entity +@Table(name = "vm_checkpoint") +public class VmCheckpointVO { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vm_id") + private long vmId; + + @Column(name = "checkpoint_id") + private String checkpointId; + + @Column(name = "create_time") + private Long createTime; + + @Column(name = "backup_id") + private Long backupId; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public VmCheckpointVO() { + } + + public VmCheckpointVO(long vmId, String checkpointId, Long createTime, Long backupId) { + this.vmId = vmId; + this.checkpointId = checkpointId; + this.createTime = createTime; + this.backupId = backupId; + } + + public long getId() { + return id; + } + + public long getVmId() { + return vmId; + } + + public String getCheckpointId() { + return checkpointId; + } + + // Creation time of the checkpoint on the host in epoch seconds, as libvirt records it. + public Long getCreateTime() { + return createTime; + } + + public Long getBackupId() { + return backupId; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public String toString() { + return String.format("VmCheckpoint {\"id\": %d, \"vmId\": %d, \"checkpointId\": \"%s\"}", id, vmId, checkpointId); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDao.java new file mode 100644 index 000000000000..14de57083970 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.VmCheckpointVO; + +import com.cloud.utils.db.GenericDao; + +public interface VmCheckpointDao extends GenericDao { + // Checkpoints of the VM in the order they were recorded, oldest first. + List listByVmId(long vmId); + + VmCheckpointVO findByVmIdAndCheckpointId(long vmId, String checkpointId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDaoImpl.java new file mode 100644 index 000000000000..dafad7e472f9 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/VmCheckpointDaoImpl.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.apache.cloudstack.backup.VmCheckpointVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class VmCheckpointDaoImpl extends GenericDaoBase implements VmCheckpointDao { + private SearchBuilder vmIdSearch; + private SearchBuilder vmIdCheckpointIdSearch; + + @PostConstruct + protected void init() { + vmIdSearch = createSearchBuilder(); + vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); + vmIdSearch.done(); + + vmIdCheckpointIdSearch = createSearchBuilder(); + vmIdCheckpointIdSearch.and("vmId", vmIdCheckpointIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); + vmIdCheckpointIdSearch.and("checkpointId", vmIdCheckpointIdSearch.entity().getCheckpointId(), SearchCriteria.Op.EQ); + vmIdCheckpointIdSearch.done(); + } + + @Override + public List listByVmId(long vmId) { + SearchCriteria sc = vmIdSearch.create(); + sc.setParameters("vmId", vmId); + // Rows are added in finalize order on the management server; the host's creation time is not + // comparable across hosts with different clocks. + return listBy(sc, new Filter(VmCheckpointVO.class, "id", true, null, null)); + } + + @Override + public VmCheckpointVO findByVmIdAndCheckpointId(long vmId, String checkpointId) { + SearchCriteria sc = vmIdCheckpointIdSearch.create(); + sc.setParameters("vmId", vmId); + sc.setParameters("checkpointId", checkpointId); + return findOneBy(sc); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 461665dfa1c1..b90b78af5e70 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -285,6 +285,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 9aaf0905786f..a8da71ca7459 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -241,3 +241,42 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.disk_offering', 'error_policy', 'VARCHAR(16) DEFAULT ''report'' COMMENT ''disk error policy (report/stop/ignore/enospace) used by KVM hypervisor'''); + +-- VM checkpoints +CREATE TABLE IF NOT EXISTS `cloud`.`vm_checkpoint`( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `vm_id` bigint unsigned NOT NULL COMMENT 'Instance ID', + `checkpoint_id` varchar(64) NOT NULL COMMENT 'Checkpoint name, used for the libvirt checkpoint and the disk bitmaps', + `create_time` bigint unsigned COMMENT 'Creation time on the host, epoch seconds', + `backup_id` bigint unsigned COMMENT 'Backup that created the checkpoint', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + INDEX `i_vm_checkpoint__vm_id`(`vm_id`), + CONSTRAINT `fk_vm_checkpoint__vm_id` FOREIGN KEY (`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- The previous checkpoint first, so that it gets the lower id. +INSERT INTO `cloud`.`vm_checkpoint` (`vm_id`, `checkpoint_id`, `create_time`, `created`) +SELECT i.`vm_id`, i.`value`, CAST(t.`value` AS UNSIGNED), NOW() +FROM `cloud`.`vm_instance_details` i + JOIN `cloud`.`vm_instance` v ON v.`id` = i.`vm_id` AND v.`removed` IS NULL + LEFT JOIN `cloud`.`vm_instance_details` t ON t.`vm_id` = i.`vm_id` AND t.`name` = 'last.checkpoint.create.time' +WHERE i.`name` = 'last.checkpoint.id' + AND NOT EXISTS (SELECT 1 FROM `cloud`.`vm_checkpoint` c + WHERE c.`vm_id` = i.`vm_id` AND c.`checkpoint_id` = i.`value` AND c.`removed` IS NULL); + +-- The active checkpoint. Skipped when it has the same id as the previous one. +INSERT INTO `cloud`.`vm_checkpoint` (`vm_id`, `checkpoint_id`, `create_time`, `created`) +SELECT i.`vm_id`, i.`value`, CAST(t.`value` AS UNSIGNED), NOW() +FROM `cloud`.`vm_instance_details` i + JOIN `cloud`.`vm_instance` v ON v.`id` = i.`vm_id` AND v.`removed` IS NULL + LEFT JOIN `cloud`.`vm_instance_details` t ON t.`vm_id` = i.`vm_id` AND t.`name` = 'active.checkpoint.create.time' +WHERE i.`name` = 'active.checkpoint.id' + AND NOT EXISTS (SELECT 1 FROM `cloud`.`vm_checkpoint` c + WHERE c.`vm_id` = i.`vm_id` AND c.`checkpoint_id` = i.`value` AND c.`removed` IS NULL); + +-- The details are not read any more. +DELETE FROM `cloud`.`vm_instance_details` +WHERE `name` IN ('active.checkpoint.id', 'active.checkpoint.create.time', + 'last.checkpoint.id', 'last.checkpoint.create.time'); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java index ddb84ab29cb3..789166ca88d1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapper.java @@ -1,79 +1,286 @@ -//Licensed to the Apache Software Foundation (ASF) under one -//or more contributor license agreements. See the NOTICE file -//distributed with this work for additional information -//regarding copyright ownership. The ASF licenses this file -//to you under the Apache License, Version 2.0 (the -//"License"); you may not use this file except in compliance -//the License. You may obtain a copy of the License at // -//http://www.apache.org/licenses/LICENSE-2.0 +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. // -//Unless required by applicable law or agreed to in writing, -//software distributed under the License is distributed on an -//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -//KIND, either express or implied. See the License for the -//specific language governing permissions and limitations -//under the License. package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuCommand; +import org.apache.cloudstack.utils.qemu.QemuImageFilename; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +// Deletes a checkpoint of the KVM backup export: the libvirt checkpoint of a running VM and the dirty bitmap of the +// same name in every layer of the disk images. A missing bitmap is not an error, so a delete can be retried. @ResourceWrapper(handles = DeleteVmCheckpointCommand.class) public class LibvirtDeleteVmCheckpointCommandWrapper extends CommandWrapper { + protected Logger logger = LogManager.getLogger(getClass()); + private static final Logger LOGGER = LogManager.getLogger(LibvirtDeleteVmCheckpointCommandWrapper.class); @Override public Answer execute(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { if (cmd.isStoppedVM()) { return deleteBitmapsOnDisks(cmd); } - return deleteDomainCheckpoint(cmd); + return deleteDomainCheckpoint(cmd, resource); } - private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd) { + // Running VM: libvirt's delete removes the bitmap from every layer of the listed disks. When it fails (forgotten + // checkpoint, a listed disk lost its bitmap) qemu is asked which disks still carry it and libvirt is told again. + private Answer deleteDomainCheckpoint(DeleteVmCheckpointCommand cmd, LibvirtComputingResource resource) { String vmName = cmd.getVmName(); String checkpointId = cmd.getCheckpointId(); - String virshCmd = String.format("virsh checkpoint-delete %s %s", vmName, checkpointId); - Script script = new Script("/bin/bash"); - script.add("-c"); - script.add(virshCmd); - String result = script.execute(); + String result = virsh("checkpoint-delete", vmName, checkpointId); + if (result == null) { + return new Answer(cmd, true, "Checkpoint deleted"); + } + logger.info("Deleting checkpoint {} of VM {} failed ({}), redefining it from the disks that carry its bitmap", checkpointId, vmName, result); + + Map diskHasBitmap; + try { + diskHasBitmap = findDisksWithBitmap(resource, vmName, checkpointId); + } catch (Exception e) { + logger.error("Failed to look up the bitmap of checkpoint {} on the disks of VM {}", checkpointId, vmName, e); + return new Answer(cmd, false, "Failed to delete checkpoint " + checkpointId + ": " + result + "; bitmap lookup failed: " + e.getMessage()); + } + List disksWithBitmap = new ArrayList<>(); + diskHasBitmap.forEach((label, has) -> { if (has) { disksWithBitmap.add(label); } }); + + if (disksWithBitmap.isEmpty()) { + // Nothing left in the images; drop libvirt's record if it still has one. + String dump = virsh("checkpoint-dumpxml", "--domain", vmName, "--checkpointname", checkpointId, "--no-domain"); + if (dump == null) { + String metadata = virsh("checkpoint-delete", vmName, checkpointId, "--metadata"); + if (metadata != null) { + return new Answer(cmd, false, "Failed to delete checkpoint " + checkpointId + ": " + metadata); + } + } else if (!dump.toLowerCase().contains("checkpoint not found")) { + return new Answer(cmd, false, "Failed to check whether libvirt still knows checkpoint " + checkpointId + ": " + dump); + } + return new Answer(cmd, true, "Checkpoint not present"); + } + + String redefine = redefineCheckpoint(vmName, checkpointId, cmd.getCheckpointCreateTime(), diskHasBitmap); + if (redefine != null) { + return new Answer(cmd, false, "Failed to redefine checkpoint " + checkpointId + " with disks " + disksWithBitmap + ": " + redefine); + } + result = virsh("checkpoint-delete", vmName, checkpointId); if (result != null) { - return new Answer(cmd, false, "Failed to delete checkpoint: " + result); + return new Answer(cmd, false, "Failed to delete checkpoint " + checkpointId + " after redefining it with disks " + disksWithBitmap + ": " + result); + } + return new Answer(cmd, true, "Checkpoint deleted, bitmap removed from disks " + disksWithBitmap); + } + + private static String virsh(String... args) { + Script script = new Script("virsh", LOGGER); + script.add(args); + return script.execute(); + } + + // Redefines the checkpoint in libvirt with the disks that carry its bitmap; a listed disk without it is refused. + static String redefineCheckpoint(String vmName, String checkpointId, Long createTime, Map diskHasBitmap) { + StringBuilder xml = new StringBuilder("\n"); + xml.append(" ").append(checkpointId).append("\n"); + // libvirt requires a creation time on a redefine but does not check it. + xml.append(" ").append(createTime != null ? createTime : System.currentTimeMillis() / 1000).append("\n"); + xml.append(" \n"); + for (Map.Entry disk : diskHasBitmap.entrySet()) { + xml.append(String.format(" \n", disk.getKey(), disk.getValue() ? "bitmap" : "no")); + } + xml.append(" \n\n"); + File xmlFile; + try { + xmlFile = File.createTempFile("checkpoint-redefine-", ".xml"); + try (FileWriter writer = new FileWriter(xmlFile)) { + writer.write(xml.toString()); + } + } catch (IOException e) { + return "could not write the checkpoint XML: " + e.getMessage(); + } + try { + return virsh("checkpoint-create", "--domain", vmName, "--xmlfile", xmlFile.getAbsolutePath(), "--redefine"); + } finally { + xmlFile.delete(); + } + } + + // Whether each disk of the running domain has the bitmap in any layer of its chain. Asked through QMP, since + // qemu-img info on an image a running qemu holds shows stale bitmap metadata. + static Map findDisksWithBitmap(LibvirtComputingResource resource, String vmName, String bitmapName) throws LibvirtException { + Map diskPathLabelMap = resource.getDiskPathLabelMap(vmName); + Domain vm = resource.getDomain(resource.getLibvirtUtilitiesHelper().getConnection(), vmName); + try { + String nodes = vm.qemuMonitorCommand(QemuCommand.buildQemuCommand("query-named-block-nodes", null), 0); + return findDisksWithBitmap(nodes, diskPathLabelMap, bitmapName); + } finally { + vm.free(); + } + } + + // Only qcow2 nodes are looked at: the protocol node of the same file has neither bitmaps nor a backing chain. + static Map findDisksWithBitmap(String namedBlockNodes, Map diskPathLabelMap, String bitmapName) { + JSONArray nodes = new JSONObject(namedBlockNodes).optJSONArray("return"); + if (nodes == null) { + throw new CloudRuntimeException("query-named-block-nodes returned no node list: " + namedBlockNodes); + } + Set filesWithBitmap = new HashSet<>(); + Map imageByFile = new LinkedHashMap<>(); + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.getJSONObject(i); + // An encrypted image is reported under a json: pseudo-filename, not its path. + String file = QemuImageFilename.plain(node.optString("file", null)); + if (file == null || !"qcow2".equals(node.optString("drv"))) { + continue; + } + JSONObject image = node.optJSONObject("image"); + if (image != null) { + imageByFile.put(file, image); + } + JSONArray bitmaps = node.optJSONArray("dirty-bitmaps"); + for (int j = 0; bitmaps != null && j < bitmaps.length(); j++) { + if (bitmapName.equals(bitmaps.getJSONObject(j).optString("name"))) { + filesWithBitmap.add(file); + } + } } - return new Answer(cmd, true, "Checkpoint deleted"); + Map diskHasBitmap = new LinkedHashMap<>(); + for (Map.Entry disk : diskPathLabelMap.entrySet()) { + boolean has = false; + for (JSONObject image = imageByFile.get(disk.getKey()); image != null; image = image.optJSONObject("backing-image")) { + if (filesWithBitmap.contains(QemuImageFilename.plain(image.optString("filename")))) { + has = true; + break; + } + } + diskHasBitmap.put(disk.getValue(), has); + } + return diskHasBitmap; } - /** - * Stopped VM: persistent bitmaps on disk images ({@code qemu-img bitmap --remove}), matching {@link LibvirtStartBackupCommandWrapper} bitmap --add. - */ + // Stopped VM: removes the persistent bitmap from every layer of each volume's chain that lists it. The template + // at the bottom of a root volume's chain never carries one and is never opened for writing. private Answer deleteBitmapsOnDisks(DeleteVmCheckpointCommand cmd) { String checkpointId = cmd.getCheckpointId(); Map diskPathUuidMap = cmd.getDiskPathUuidMap(); if (diskPathUuidMap == null || diskPathUuidMap.isEmpty()) { return new Answer(cmd, false, "No disks provided for bitmap removal"); } - for (Map.Entry entry : diskPathUuidMap.entrySet()) { - String diskPath = entry.getKey(); - Script script = new Script("qemu-img"); - script.add("bitmap"); - script.add("--remove"); - script.add(diskPath); - script.add(checkpointId); - String result = script.execute(); - if (result != null) { - return new Answer(cmd, false, - "Failed to remove bitmap " + checkpointId + " from disk " + diskPath + ": " + result); + Map diskPathPassphraseMap = cmd.getDiskPathPassphraseMap() != null ? cmd.getDiskPathPassphraseMap() : Collections.emptyMap(); + List removedFrom = new ArrayList<>(); + for (String diskPath : diskPathUuidMap.keySet()) { + byte[] passphrase = diskPathPassphraseMap.get(diskPath); + List layers; + try { + layers = queryChainLayersWithBitmap(diskPath, checkpointId); + } catch (Exception e) { + return new Answer(cmd, false, "Failed to read the backing chain of disk " + diskPath + ": " + e.getMessage()); + } + for (String layer : layers) { + String result; + try { + result = removeBitmapFromImage(layer, passphrase, checkpointId); + } catch (IOException e) { + return new Answer(cmd, false, "Failed to create the key file for removing bitmap " + checkpointId + " from disk " + layer); + } + if (result == null) { + removedFrom.add(layer); + } else if (result.contains(String.format("Dirty bitmap '%s' not found", checkpointId))) { + logger.info("Bitmap {} is not present on {}: {}", checkpointId, layer, result); + } else { + return new Answer(cmd, false, "Failed to remove bitmap " + checkpointId + " from disk " + layer + ": " + result); + } + } + } + return new Answer(cmd, true, removedFrom.isEmpty() ? "Checkpoint not present" : "Checkpoint bitmap removed from " + removedFrom); + } + + // The files of the volume's backing chain, top first, whose bitmap list contains the name. + private List queryChainLayersWithBitmap(String diskPath, String bitmapName) { + Script script = new Script("qemu-img", logger); + script.add("info", "-U", "--backing-chain", "--output=json", diskPath); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = script.execute(parser); + if (result != null) { + throw new CloudRuntimeException(result); + } + return findChainLayersWithBitmap(parser.getLines(), bitmapName); + } + + static List findChainLayersWithBitmap(String backingChainInfo, String bitmapName) { + List layers = new ArrayList<>(); + // Script merges stderr into stdout, so a qemu-img warning may precede the JSON. + int start = backingChainInfo.indexOf('['); + if (start < 0) { + throw new CloudRuntimeException("qemu-img info returned no JSON: " + backingChainInfo); + } + JSONArray chain = new JSONArray(backingChainInfo.substring(start)); + for (int i = 0; i < chain.length(); i++) { + JSONObject image = chain.getJSONObject(i); + JSONObject specific = image.optJSONObject("format-specific"); + JSONObject data = specific != null ? specific.optJSONObject("data") : null; + JSONArray bitmaps = data != null ? data.optJSONArray("bitmaps") : null; + for (int j = 0; bitmaps != null && j < bitmaps.length(); j++) { + if (bitmapName.equals(bitmaps.getJSONObject(j).optString("name"))) { + layers.add(image.getString("filename")); + break; + } } } - return new Answer(cmd, true, "Checkpoint bitmap removed from disks"); + return layers; + } + + private String removeBitmapFromImage(String diskPath, byte[] passphrase, String bitmapName) throws IOException { + Script script = new Script("qemu-img", logger); + script.add("bitmap", "--remove"); + if (passphrase == null || passphrase.length == 0) { + script.add(diskPath, bitmapName); + return script.execute(); + } + // The key file holds the plaintext passphrase, so it only lives for the duration of the call. + try (KeyFile key = new KeyFile(passphrase)) { + script.add("--object", String.format("secret,id=sec0,file=%s", key)); + script.add("--image-opts", String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); + script.add(bitmapName); + return script.execute(); + } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 629fd179afb2..6abbb5d6eb70 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.backup.StartBackupCommand; import org.apache.cloudstack.utils.cryptsetup.KeyFile; import org.apache.cloudstack.utils.qemu.QemuCommand; +import org.apache.cloudstack.utils.qemu.QemuImageFilename; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.json.JSONArray; @@ -61,7 +62,7 @@ public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingReso try { if (StringUtils.isNotBlank(fromCheckpointId)) { - Answer redefineAnswer = ensureFromCheckpointExists(cmd, fromCheckpointId, fromCheckpointCreateTime); + Answer redefineAnswer = ensureFromCheckpointExists(cmd, fromCheckpointId, fromCheckpointCreateTime, resource); if (redefineAnswer != null) { return redefineAnswer; } @@ -111,7 +112,10 @@ public Answer handleRunningVmBackup(StartBackupCommand cmd, LibvirtComputingReso } } - private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromCheckpointId, Long fromCheckpointCreateTime) { + // CloudStack domains are transient, so a stop/start forgets every checkpoint while the bitmaps stay in the images. + // A forgotten from-checkpoint is redefined with the disks that carry its bitmap, so libvirt can later delete it. + private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromCheckpointId, Long fromCheckpointCreateTime, + LibvirtComputingResource resource) { String vmName = cmd.getVmName(); Script dumpScript = new Script("/bin/bash"); dumpScript.add("-c"); @@ -124,40 +128,24 @@ private Answer ensureFromCheckpointExists(StartBackupCommand cmd, String fromChe return new StartBackupAnswer(cmd, false, "From checkpoint create time is null for checkpoint " + fromCheckpointId); } - String redefineXml = createCheckpointXmlForRedefine(fromCheckpointId, fromCheckpointCreateTime); - File redefineFile; + Map diskHasBitmap; try { - redefineFile = File.createTempFile("checkpoint-redefine-", ".xml"); + diskHasBitmap = LibvirtDeleteVmCheckpointCommandWrapper.findDisksWithBitmap(resource, vmName, fromCheckpointId); } catch (Exception e) { - return new StartBackupAnswer(cmd, false, "Failed to create temp file for checkpoint redefine: " + e.getMessage()); + return new StartBackupAnswer(cmd, false, "Failed to look up the bitmap of from-checkpoint " + fromCheckpointId + + " on the disks: " + e.getMessage()); } - try (FileWriter writer = new FileWriter(redefineFile)) { - writer.write(redefineXml); - } catch (Exception e) { - redefineFile.delete(); - return new StartBackupAnswer(cmd, false, "Failed to write checkpoint redefine XML: " + e.getMessage()); + if (!diskHasBitmap.containsValue(Boolean.TRUE)) { + logger.warn("No disk of VM {} carries the bitmap of from-checkpoint {}; the backup will be a full one", vmName, fromCheckpointId); + return null; } - String createCmd = String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, redefineFile.getAbsolutePath()); - Script createScript = new Script("/bin/bash"); - createScript.add("-c"); - createScript.add(createCmd); - String result = createScript.execute(); - redefineFile.delete(); + String result = LibvirtDeleteVmCheckpointCommandWrapper.redefineCheckpoint(vmName, fromCheckpointId, fromCheckpointCreateTime, diskHasBitmap); if (result != null) { return new StartBackupAnswer(cmd, false, "Failed to redefine from-checkpoint " + fromCheckpointId + ": " + result); } return null; } - private String createCheckpointXmlForRedefine(String checkpointName, Long createTime) { - StringBuilder xml = new StringBuilder(); - xml.append("\n"); - xml.append(" ").append(checkpointName).append("\n"); - xml.append(" ").append(createTime).append("\n"); - xml.append(""); - return xml.toString(); - } - private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, String socket, LibvirtComputingResource resource) throws LibvirtException { StringBuilder xml = new StringBuilder(); xml.append("\n"); @@ -228,23 +216,24 @@ private Answer handleStoppedVmBackup(StartBackupCommand cmd, String toCheckpoint script.add("bitmap"); script.add("--add"); + String result; if (passphrase != null && passphrase.length > 0) { - KeyFile srcKey; - try { - srcKey = new KeyFile(passphrase); + // The key file holds the plaintext passphrase, so it only lives for the duration of the call. + try (KeyFile srcKey = new KeyFile(passphrase)) { + script.add("--object"); + script.add(String.format("secret,id=sec0,file=%s", srcKey)); + script.add("--image-opts"); + script.add(String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); + script.add(toCheckpointId); + result = script.execute(); } catch (IOException ex) { return new StartBackupAnswer(cmd, false, "Failed to create KeyFile while adding bitmap " + toCheckpointId + " to disk " + diskPath); } - script.add("--object"); - script.add(String.format("secret,id=sec0,file=%s", srcKey)); - script.add("--image-opts"); - script.add(String.format("driver=qcow2,file.driver=file,file.filename=%s,encrypt.key-secret=sec0", diskPath)); } else { script.add(diskPath); + script.add(toCheckpointId); + result = script.execute(); } - - script.add(toCheckpointId); - String result = script.execute(); if (result != null) { return new StartBackupAnswer(cmd, false, "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); @@ -274,7 +263,8 @@ private Map getVmDiskPathHasFromCheckpointMap(Domain vm, String if (inserted == null) { continue; } - String file = inserted.optString("file"); + // An encrypted image is reported under a json: pseudo-filename, not its path. + String file = QemuImageFilename.plain(inserted.optString("file")); if (StringUtils.isBlank(file)) { continue; } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageFilename.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageFilename.java new file mode 100644 index 000000000000..f658c6cdda12 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImageFilename.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.utils.qemu; + +import org.json.JSONException; +import org.json.JSONObject; + +// qemu reports an image opened with options that have no place in a plain path, such as the key secret of an +// encrypted qcow2, under a json:{...} pseudo-filename whose innermost file.filename is the path. +public final class QemuImageFilename { + private static final String JSON_PREFIX = "json:"; + + private QemuImageFilename() { + } + + // The plain path behind a qemu file name: the innermost file.filename of a json: pseudo-filename, else the name. + public static String plain(String filename) { + if (filename == null || !filename.startsWith(JSON_PREFIX)) { + return filename; + } + try { + JSONObject options = new JSONObject(filename.substring(JSON_PREFIX.length())); + while (options.has("file") && options.optJSONObject("file") != null) { + options = options.getJSONObject("file"); + } + String plain = options.optString("filename", null); + return plain != null ? plain : filename; + } catch (JSONException e) { + return filename; + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapperTest.java index 769fdc9aa7b6..90ab2d66baea 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVmCheckpointCommandWrapperTest.java @@ -14,75 +14,447 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.BufferedReader; +import java.io.File; +import java.io.StringReader; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.cloudstack.backup.DeleteVmCheckpointCommand; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) public class LibvirtDeleteVmCheckpointCommandWrapperTest { + // vda is an overlay on /p/base.qcow2 (a VM snapshot): cp-5 in the top layer, cp-4 only in the base; vdb has no + // bitmaps. Each disk has a format node and a storage node with the same file, as libvirt builds them. + private static final String NAMED_NODES = "{\"return\": [" + + "{\"node-name\": \"libvirt-2-format\", \"file\": \"/p/vda.qcow2\", \"drv\": \"qcow2\", \"ro\": false," + + " \"image\": {\"filename\": \"/p/vda.qcow2\", \"backing-image\": {\"filename\": \"/p/base.qcow2\"}}," + + " \"dirty-bitmaps\": [{\"name\": \"cp-5\"}]}," + + "{\"node-name\": \"libvirt-2-storage\", \"file\": \"/p/vda.qcow2\", \"drv\": \"file\", \"ro\": false, \"image\": {\"filename\": \"/p/vda.qcow2\"}}," + + "{\"node-name\": \"libvirt-3-format\", \"file\": \"/p/base.qcow2\", \"drv\": \"qcow2\", \"ro\": true," + + " \"image\": {\"filename\": \"/p/base.qcow2\"}, \"dirty-bitmaps\": [{\"name\": \"cp-4\"}, {\"name\": \"cp-5\"}]}," + + "{\"node-name\": \"libvirt-3-storage\", \"file\": \"/p/base.qcow2\", \"drv\": \"file\", \"ro\": false, \"image\": {\"filename\": \"/p/base.qcow2\"}}," + + "{\"node-name\": \"libvirt-1-format\", \"file\": \"/p/vdb.qcow2\", \"drv\": \"qcow2\", \"ro\": false," + + " \"image\": {\"filename\": \"/p/vdb.qcow2\"}, \"dirty-bitmaps\": []}," + + "{\"node-name\": \"libvirt-1-storage\", \"file\": \"/p/vdb.qcow2\", \"drv\": \"file\", \"ro\": false, \"image\": {\"filename\": \"/p/vdb.qcow2\"}}" + + "]}"; + private static final Map DISK_LABELS = new LinkedHashMap<>(Map.of("/p/vda.qcow2", "vda")); + static { + DISK_LABELS.put("/p/vdb.qcow2", "vdb"); + } + + // qemu-img info --backing-chain of a stopped volume whose overlay has no bitmap and whose base has cp-2. + private static final String CHAIN_INFO = "[" + + "{\"filename\": \"/p/vol.qcow2\", \"format\": \"qcow2\", \"backing-filename\": \"/p/old.qcow2\"," + + " \"format-specific\": {\"type\": \"qcow2\", \"data\": {\"compat\": \"1.1\"}}}," + + "{\"filename\": \"/p/old.qcow2\", \"format\": \"qcow2\", \"backing-filename\": \"/p/template.qcow2\"," + + " \"format-specific\": {\"type\": \"qcow2\", \"data\": {\"bitmaps\": [{\"name\": \"cp-1\"}, {\"name\": \"cp-2\"}]}}}," + + "{\"filename\": \"/p/template.qcow2\", \"format\": \"qcow2\", \"format-specific\": {\"type\": \"qcow2\", \"data\": {}}}" + + "]"; + private static final String JSON_OUTPUT = "JSON:"; + private LibvirtDeleteVmCheckpointCommandWrapper wrapper; private DeleteVmCheckpointCommand command; + private LibvirtComputingResource resource; + private Domain domain; @Before - public void setUp() { + public void setUp() throws Exception { wrapper = new LibvirtDeleteVmCheckpointCommandWrapper(); command = Mockito.mock(DeleteVmCheckpointCommand.class); + resource = Mockito.mock(LibvirtComputingResource.class); + domain = Mockito.mock(Domain.class); + LibvirtUtilitiesHelper helper = Mockito.mock(LibvirtUtilitiesHelper.class); + Connect conn = Mockito.mock(Connect.class); + Mockito.lenient().when(resource.getLibvirtUtilitiesHelper()).thenReturn(helper); + Mockito.lenient().when(helper.getConnection()).thenReturn(conn); + Mockito.lenient().when(resource.getDomain(conn, "i-2-3-VM")).thenReturn(domain); + Mockito.lenient().when(resource.getDiskPathLabelMap("i-2-3-VM")).thenReturn(DISK_LABELS); + } + + private void runningVmCommand(String checkpointId) { + Mockito.when(command.isStoppedVM()).thenReturn(false); + Mockito.when(command.getVmName()).thenReturn("i-2-3-VM"); + Mockito.when(command.getCheckpointId()).thenReturn(checkpointId); + } + + private void stoppedVmCommand(String checkpointId, String diskPath) { + Mockito.when(command.isStoppedVM()).thenReturn(true); + Mockito.when(command.getCheckpointId()).thenReturn(checkpointId); + Mockito.when(command.getDiskPathUuidMap()).thenReturn(Map.of(diskPath, "vol-1")); + } + + private void qmpReturns(String namedBlockNodes) throws LibvirtException { + Mockito.when(domain.qemuMonitorCommand(Mockito.contains("query-named-block-nodes"), Mockito.eq(0))).thenReturn(namedBlockNodes); + } + + // Each Script gets the next result; its arguments, plus the content of an --xmlfile (deleted afterwards), are + // recorded in order. A result starting with JSON_OUTPUT feeds the parser of execute(parser) and counts as success. + private static MockedConstruction