Articles - Live backup of KVM VM images under RHEL8/9

Publication date: 11/16/22

I've created the following script to back up VMs running on top of KVM under RHEL9. I believe it's also compatible with RHEL8, but not tested. It utilizes the recently introduced "backup-begin" function and requires libvirt 7.2+ and QEMU 4.2+. The main advantage (compared to the old method with snapshots) is that it doesn't pause the existing VMs. Besides, I've read about cases when snapshots caused VM image corruption, so certainly preferred to stay away from it. Documentation

#!/bin/bash

# Doc: https://libvirt.org/kbase/live_full_disk_backup.html
# Partially based on https://nixlab.org/blog/backup-kvm-virtual-machines

################
### SETTINGS ###
################

# Backup dir
BACKUP_DIR="/data/_backup"

# Retention policy
RETENTION=3 # how many copies of each config/image to keep locally

# Timestamp
TIMESTAMP="$(date +%Y%m%d%H%M%S)"

# target vm list (running)
VM_LIST=$(virsh list | grep running | awk '{print $2}' | tr 'n' ' ')

# Log file
LOGFILE="/var/log/kvmbackup.log"
STDOUT=1 # 0 - standard output disabled, 1 - enabled, 2 - debug level (TBD)
LOG=1 # 0 - logging disabled, 1 - enabled, 2 - debug level (TBD)

#################
### FUNCTIONS ###
#################

function print_log {
  local RET=$? # have to keep previous error code and return it
  local MSG="$1"
  local LSTDOUT="${2:-$STDOUT}" # Allow stdout and logging override when calling this function
  local LLOG="${3:-$LOG}"
  local CUR_TS="$(date '+%Y%m%d %H:%I:%S')"
  [ ${LSTDOUT} -ne 0 ] && echo -e "${CUR_TS} - ${MSG}"
  [ ${LLOG} -ne 0 ] && echo -e "${CUR_TS} - ${MSG}" >> ${LOGFILE}
  return ${RET}
}

function die {
  print_log "ERROR: $1" 1 1
  exit
}

function create_folders {
  while [ -n "$1" ]; do
    if [ ! -d "$1" ]; then
      mkdir -p "$1" && print_log "Folder "$1" was successfully created" || die "Cannot create $1 folder"
    fi
    shift
  done
}

function purge_expired {
  local LDIR="$1"
  local LNAMEREGEX="$2"
  local LRET="$3"
  local FILES2DEL="$(ls -1t ${LDIR}/${LNAMEREGEX} | tail -n +$((LRET+1)))"
  local f
  if [ -n "${FILES2DEL}" ]; then
    for f in ${FILES2DEL}; do
      if [ -f "${f}" ]; then
        rm -f "${f}" && print_log "Expired file ${f} was purged" || die "Can't purge expired ${f} file"
      fi
    done
  fi
}

#################
### MAIN CODE ###
#################

[ $(rpm -q libvirt | egrep -c 'libvirt-(7.[2-9]|8|9)') -eq 0 ] && die "Sorry, this script requires libvirt 7.2+"
[ $(rpm -q qemu-kvm | egrep -c 'qemu-kvm-(4.[2-9]|[5-9])') -eq 0 ] && die "Sorry, this script requires QEMU 4.2+"

print_log "=== START BACKUP OF THE HOST: $(hostname) ==="
print_log "Running VMs to be backed up: ${VM_LIST}"

for VM in ${VM_LIST}; do
  print_log "Start backup of VM: ${VM}"
  BACKUP_CFG="${BACKUP_DIR}/${VM}/config"
  BACKUP_IMG="${BACKUP_DIR}/${VM}/images"
  create_folders ${BACKUP_CFG} ${BACKUP_IMG} # Create backup folders if don't exist

  # Dump VM configuration to XML file
  OUTXML="${BACKUP_CFG}/${VM}.xml.${TIMESTAMP}"
  virsh dumpxml "${VM}" > "${OUTXML}" && print_log "Dumped XML config for "${VM}" VM to ${OUTXML} file" || die "Failed to dump XML config for "${VM}" VM to ${OUTXML} file"
  purge_expired "${BACKUP_CFG}" "${VM}.xml.*" "${RETENTION}"

  # Start VM image backups and wait for their completion
  virsh backup-begin "${VM}" >/dev/null 2>&1 && print_log "Backup job for the "${VM}" VM has been started" || die "Failed to start backup job for "${VM}" VM"
  while [ "$(virsh domjobinfo "${VM}" | grep '^Job type:' | awk '{ print $3 }')" != "None" ]; do sleep 5; done
  print_log "Backup job for the "${VM}" VM has finished"

  # Move backup images to the backup subfolder
  VM_IMAGES=$(virsh domblklist "${VM}" | egrep '[vs]d[a-z].*/' | awk '{print $2}')
  if [ -n "${VM_IMAGES}" ]; then
    for VM_IMG in ${VM_IMAGES}; do
      BKP_LIST=$(ls -1 ${VM_IMG}.* 2>/dev/null)
      if [ -n "${BKP_LIST}" ]; then
        for BKP_IMG in $(ls -1 ${VM_IMG}.* 2>/dev/null); do
          print_log "Found backup image ${BKP_IMG}"
          BKP_IMG_TS="${BKP_IMG}.${TIMESTAMP}"
          # Add human-readable timestamp to the image name and then move it to backup folder
          mv -f "${BKP_IMG}" "${BKP_IMG_TS}" || die "Failed to rename "${BKP_IMG}" image to "${BKP_IMG_TS}""
          mv -f "${BKP_IMG_TS}" "${BACKUP_IMG}" && print_log "Moved "${BKP_IMG_TS}" image to "${BACKUP_IMG}" folder" || die "Failed to move "${BKP_IMG_TS}" image to "${BACKUP_IMG}" folder"
        done
      else
        print_log "WARNING: No backup images were found for the main ${VM_IMG} image, skipping it"
      fi
      purge_expired "${BACKUP_IMG}" "$(basename ${VM_IMG}).*" "${RETENTION}"
    done
  else
    die "No images were reported by "virsh domblklist ${VM}" command, something is heavily wrong"
  fi
  print_log "End backup of VM: ${VM}"
done

# Cleanup
FILES_TO_DELETE="$(find ${BACKUP_DIR} -type f -mtime +${RETENTION})"
if [ $(find ${BACKUP_DIR} -type f -mtime +${RETENTION} | wc -l) -gt 1 ]; then
  print_log "The following copies have expired and will be deleted:n$(find ${BACKUP_DIR} -type f -mtime +${RETENTION})"
  find ${BACKUP_DIR} -type f -mtime +${RETENTION} -delete && print_log "Expired copies were successfully deleted" || die "Could not delete expired copies"
fi

print_log "=== END BACKUP OF THE HOST: $(hostname) ==="

20221116 12:12:28 - === START BACKUP OF THE HOST: kvm ===
20221116 12:12:28 - Running VMs to be backed up: int
20221116 12:12:28 - Start backup of VM: int
20221116 12:12:28 - Dumped XML config for "int" VM to /data/_backup/int/config/int.xml.20221116124228 file
20221116 12:12:28 - Expired file /data/_backup/int/config/int.xml.20221116120424 was purged
20221116 12:12:28 - Backup job for the "int" VM has been started
20221116 12:12:04 - Backup job for the "int" VM has finished
20221116 12:12:04 - Found backup image /data/kvm/images/int.img.1668620548
20221116 12:12:04 - Moved "/data/kvm/images/int.img.1668620548.20221116124228" image to "/data/_backup/int/images" folder
20221116 12:12:04 - Expired file /data/_backup/int/images/int.img.1668618264.20221116120424 was purged
20221116 12:12:04 - Found backup image /data/kvm/images/int-data.img.1668620548
20221116 12:12:04 - Moved "/data/kvm/images/int-data.img.1668620548.20221116124228" image to "/data/_backup/int/images" folder
20221116 12:12:04 - Expired file /data/_backup/int/images/int-data.img.1668618264.20221116120424 was purged
20221116 12:12:04 - End backup of VM: int
20221116 12:12:04 - === END BACKUP OF THE HOST: kvm ===