commit 228185bd18245e60351a2b5131b324d64ff6cdf4 Author: Dmitry Vlasov Date: Thu Dec 7 08:11:01 2023 +0300 Initial commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..44b4224 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* eol=lf \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4fd29bf --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# Borgbackup Test + +### Example of use +``` +git clone https://gitlab.resds.ru/vlasov.dv/borgbackup-test.git +cd borgbackup-test +chmod +x create-vm.sh +./create-vm.sh --bootstrap-env env +vi env +sudo ./create-vm.sh -e ./env +``` + +### Usage of `./create-vm.sh`: +``` +usage: + ./create-vm.sh required_args [optional_args] OR + ./create-vm.sh misc_args + +Create a new virtual machine according to specified parameters and run +test on filesystem + +misc_args: + --bootstrap-env ENV_PATH create template env file and exit + +required_args: + -n, --name NAME name of a virtual machine + -o, --os-variant OS_VARIANT operating system + -m, --memory MEMORY RAM size + -c, --vcpus VCPUS Amount of of cores + -i, --image IMAGE URL or path to an ISO image + -s, --size SIZE Size of an OS drive (example: 8G) + -b, --bridge BRIDGE bridge name (LinuxBridge only) + -a, --ip-address IP ip address of eth0 + -p, --prefix PREFIX network prefix + -g, --gateway GATEWAY ip address of network gateway + -d, --dns DNS ip address of dns server + -S, --source-dev SOURCE_DEV size of the created image for + performing a test on it + -f, --fstype FSTYPE filesystem type + default: xfs + +optional_args: + -h, --help show this help message and exit + -e, --env export args from env file + -r, --rm-if-exist if vm exist, remove it first + default: false + -P, --pubkey PUBKEY place public key into autorized_keys of + all defined in config.yaml users + default: /home/$USER/.ssh/id_rsa.pub + -v, --vnc-password VNC_PASSWORD vnc console password + default: PASSWD1! + -C, --cloud-init CLOUD_INIT Path to a cloud-init config - will be + created if not provided + +env example: + NAME=Ubuntu22.04 + OS_VARIANT=ubuntu22.04 + MEMORY=5000 + VCPUS=2 + IMAGE=https://cloud-images.ubuntu.com/jammy/20231012/jammy-server-cloudimg-amd64.img + SIZE=8G + BRIDGE=virbr0 + IP=192.168.122.2 + PREFIX=24 + GATEWAY=192.168.122.1 + DNS=8.8.8.8 + SOURCE_DEV=/dev/vdb # CHANGE IT! + FSTYPE=xfs + RM=true + PUBKEY=/root/.ssh/id_rsa.pub + VNC_PASSWORD=PASSWD1! + + +cloud-init example: + #cloud-config + package_upgrade: false + packages: + - vim + - htop + - borgbackup + users: + - name: ubuntu + ssh-authorized-keys: + - {{PUBKEY}} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + ssh_pwauth: true + lock-passwd: false + chpasswd: {expire: False} + groups: sudo + shell: /bin/bash + - name: adminuser + ssh-authorized-keys: + - {{PUBKEY}} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + ssh_pwauth: true + lock-passwd: false + chpasswd: {expire: False} + groups: sudo + shell: /bin/bash + runcmd: + - wipefs -a /dev/{{DEV_INTERNAL_NAME}} + - mkfs.{{FSTYPE}} /dev/{{DEV_INTERNAL_NAME}} + - mkdir -p {{TESTDIR}} + - mount /dev/{{DEV_INTERNAL_NAME}} {{TESTDIR}} + - borg init -e none {{TESTDIR}} + - dd if=/dev/urandom bs=2G count=1 | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_A - --stdin-name test + - dd if=/dev/urandom bs=2G count=1 | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_B - --stdin-name test + - dd if=/dev/urandom bs=2G count=1 | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_C - --stdin-name test +``` \ No newline at end of file diff --git a/create-vm.sh b/create-vm.sh new file mode 100644 index 0000000..b25222f --- /dev/null +++ b/create-vm.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env bash +# Creating a new virtual (another virt-viewer wrapper, yes) +# +# == Defaults ================================================================= +DEFAULT_DEV_INTERNAL_NAME=vdb +DEFAULT_BASE_PATH=/tmp/create-vm +DEFAULT_TESTDIR="/fs-test" +DEFAULT_USER_INIT="#cloud-config +package_upgrade: false +packages: +- vim +- htop +- borgbackup +users: +- name: ubuntu + ssh-authorized-keys: + - {{PUBKEY}} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + ssh_pwauth: true + lock-passwd: false + chpasswd: {expire: False} + groups: sudo + shell: /bin/bash +- name: adminuser + ssh-authorized-keys: + - {{PUBKEY}} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + ssh_pwauth: true + lock-passwd: false + chpasswd: {expire: False} + groups: sudo + shell: /bin/bash +runcmd: +- wipefs -a /dev/{{DEV_INTERNAL_NAME}} +- mkfs.{{FSTYPE}} /dev/{{DEV_INTERNAL_NAME}} +- mkdir -p {{TESTDIR}} +- mount /dev/{{DEV_INTERNAL_NAME}} {{TESTDIR}} +- borg init -e none {{TESTDIR}} +- dd if=/dev/urandom bs=2G count=1 \ + | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_A - --stdin-name test +- dd if=/dev/urandom bs=2G count=1 \ + | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_B - --stdin-name test +- dd if=/dev/urandom bs=2G count=1 \ + | borg create -v -C zstd --stats {{TESTDIR}}::TESTARCHIVE_C - --stdin-name test +" +DEFAULT_NETPLAN="network: + version: 2 + renderer: networkd + ethernets: + enp1s0: + dhcp4: false + addresses: + - {{IP}}/{{PREFIX}} + routes: + - to: default + via: {{GATEWAY}} + nameservers: + addresses: + - {{DNS}} +" +DEFAULT_ENV="NAME=Ubuntu22.04 +OS_VARIANT=ubuntu22.04 +MEMORY=5000 +VCPUS=2 +IMAGE=https://cloud-images.ubuntu.com/jammy/20231012/jammy-server-cloudimg-amd64.img +SIZE=8G +BRIDGE=virbr0 +IP=192.168.122.2 +PREFIX=24 +GATEWAY=192.168.122.1 +DNS=8.8.8.8 +SOURCE_DEV=/dev/vdb # CHANGE IT! +FSTYPE=xfs +RM=true +PUBKEY=/root/.ssh/id_rsa.pub +VNC_PASSWORD=PASSWD1! +" +# == Help ===================================================================== +HELP=" +usage: + $0 required_args [optional_args] OR + $0 misc_args + +Create a new virtual machine according to specified parameters and run +test on filesystem + +misc_args: + --bootstrap-env ENV_PATH create template env file and exit + +required_args: + -n, --name NAME name of a virtual machine + -o, --os-variant OS_VARIANT operating system + -m, --memory MEMORY RAM size + -c, --vcpus VCPUS Amount of of cores + -i, --image IMAGE URL or path to an ISO image + -s, --size SIZE Size of an OS drive (example: 8G) + -b, --bridge BRIDGE bridge name (LinuxBridge only) + -a, --ip-address IP ip address of eth0 + -p, --prefix PREFIX network prefix + -g, --gateway GATEWAY ip address of network gateway + -d, --dns DNS ip address of dns server + -S, --source-dev SOURCE_DEV size of the created image for + performing a test on it + -f, --fstype FSTYPE filesystem type + default: xfs + +optional_args: + -h, --help show this help message and exit + -e, --env export args from env file + -r, --rm-if-exist if vm exist, remove it first + default: false + -P, --pubkey PUBKEY place public key into autorized_keys of + all defined in config.yaml users + default: /home/\$USER/.ssh/id_rsa.pub + -v, --vnc-password VNC_PASSWORD vnc console password + default: PASSWD1! + -C, --cloud-init CLOUD_INIT Path to a cloud-init config - will be + created if not provided + +env example: +$(echo -e "$DEFAULT_ENV"|paste /dev/null -|expand -2) + +cloud-init example: +$(echo -e "$DEFAULT_USER_INIT"|paste /dev/null -|expand -2) +" + +# == Functions ================================================================ +function help() { + echo -ne "${HELP/[$'\n']}" + echo +} + +function required() { + [[ -n "$1" ]] || { + echo "Incorrect options provided" + help + exit 1 + } +} + +function bootstrap_env() { + required $1 + echo -e "$DEFAULT_ENV" > $1 +} + +function rm_domain() { + virsh undefine --nvram $2 + virsh destroy $2 + rm -f $1/backing.img + rm -f $1/cidata.iso + rm -f $1/meta-data + rm -f $1/network-config + rm -f $1/Ubuntu22.04.img + rm -f $1/Ubuntu22.04.qcow2 + rm -f $1/user-data +} + +function cleanup_known_host() { + ssh-keygen -f "/root/.ssh/known_hosts" -R "$1" +} + +# == Args parse =============================================================== +options=$(getopt -n $0 -o n:o:m:c:i:s:b:a:p:g:d:S:f:he:rP:v:C: \ + --long bootstrap-env: \ + --long name: \ + --long os-variant: \ + --long memory: \ + --long vcpus: \ + --long image: \ + --long size: \ + --long bridge: \ + --long ip-address: \ + --long prefix: \ + --long gateway: \ + --long dns: \ + --long source-dev: \ + --long fstype: \ + --long help \ + --long env: \ + --long rm-if-exist \ + --long pubkey: \ + --long vnc-password: \ + --long cloud-init: \ + -- "$@") +[ $? -eq 0 ] || { + echo "Incorrect options provided" + help + exit 1 +} + +eval set -- "$options" +while true; do + case $1 in + --bootstrap-env) + shift + bootstrap_env $1 + exit 0 + ;; + -n | --name) + shift + NAME=$1 + shift + ;; + -o | --os-variant) + shift + OS_VARIANT=$1 + shift + ;; + -m | --memory) + shift + MEMORY=$1 + shift + ;; + -c | --vcpus) + shift + VCPUS=$1 + shift + ;; + -i | --image) + shift + IMAGE=$1 + shift + ;; + -s | --size) + shift + SIZE=${1:-"8G"} + shift + ;; + -b | --bridge) + shift + BRIDGE=$1 + shift + ;; + -p | --prefix) + shift + PREFIX=$1 + shift + ;; + -a | --ip-address) + shift + IP=$1 + shift + ;; + -g | --gateway) + shift + GATEWAY=$1 + shift + ;; + -d | --dns) + shift + DNS=$1 + shift + ;; + -S | --source-dev) + shift + SOURCE_DEV=$1 + shift + ;; + -f | --fstype) + shift + FSTYPE=${1:-"xfs"} + shift + ;; + -h | --help) + help + exit 0 + ;; + -e | --env) + shift + ENV=$1 + shift + ;; + -r | --rm-if-exist) + shift + RM=${1:-false} + shift + ;; + -p | --pubkey) + shift + PUBKEY=${1:-"/root/.ssh/id_rsa.pub"} + shift + ;; + -v | --vnc-password) + shift + VNC_PASSWORD=${1:-"PASSWD1!"} + shift + ;; + -C | --cloud-init) + shift + CLOUD_INIT=$1 + shift + ;; + --) + shift + break + esac +done + +# Required args +if [[ -z "$ENV" ]]; then + required $NAME + required $OS_VARIANT + required $MEMORY + required $VCPUS + required $IMAGE + required $BRIDGE + required $IP + required $PREFIX + required $GATEWAY + required $DNS + required $SOURCE_DEV +fi + +if [[ -n "$ENV" ]]; then + if [[ $ENV != /* ]] || [[ $ENV != ./* ]] || [[ $ENV != ./* ]]; then + ENV="./$ENV" + fi + if [[ -e "$ENV" ]]; then + set -a + source $ENV + set +a + else + echo Env file $ENV does not exist + exit 1 + fi +fi + +IMAGE_IS_URL=false +if [[ "$IMAGE" == https://* ]] || [[ "$IMAGE" == http://* ]]; then + IMAGE_IS_URL=true +fi + +CP_CLOUD_INIT=false +if [[ -n "$CLOUD_INIT" ]] && [[ -e "$CLOUD_INIT" ]]; then + CP_CLOUD_INIT=true +fi + +ENV=$(readlink -f "$ENV") +PUBKEY=$(readlink -f "$PUBKEY") +CLOUD_INIT=$(readlink -f "$CLOUD_INIT") + +BASE_PATH=$DEFAULT_BASE_PATH/$NAME + +# == Dependencies ============================================================= +DEBIAN_FRONTEND=noninteractive apt-get install -y genisoimage \ + qemu-kvm libvirt-daemon-system virtinst + +# == Asserts ================================================================== +printf "virsh " +if ! virsh --version; then + echo "Please install libvirt packages first" + exit 1 +fi + +printf "virt-install " +if ! virt-install --version; then + echo "Please install virt-install package first" + exit 1 +fi + +if ! genisoimage --version; then + echo "Please install genisoimage package first" + exit 1 +fi + +if ! getopt --version; then + echo "Please install getopt package first" + exit 1 +fi + +if [[ ! -e $PUBKEY ]]; then + ssh-keygen -t rsa -f ${PUBKEY//".pub"/""} -q -P "" +fi + +# == Main ===================================================================== +# Remove if exist +if [ "$RM" = true ]; then + rm_domain $BASE_PATH $NAME 2>/dev/null +fi +mkdir -p $BASE_PATH + +# Preparing user-data +if [[ "$CP_CLOUD_INIT" == true ]]; then + cp -f $CLOUD_INIT $BASE_PATH/user-data +else + + user_init=${DEFAULT_USER_INIT//"{{PUBKEY}}"/"$(cat $PUBKEY 2>/dev/null)"} + + user_init=${user_init//"{{FSTYPE}}"/"$FSTYPE"} + user_init=${user_init//"{{DEV_INTERNAL_NAME}}"/"$DEFAULT_DEV_INTERNAL_NAME"} + + user_init=${user_init//"{{TESTDIR}}"/"$DEFAULT_TESTDIR"} + + echo -e "$user_init" > $BASE_PATH/user-data +fi + +# Preparing meta-data +echo -e "local-hostname: $NAME" > $BASE_PATH/meta-data + +# Preparing network-config +netplan=${DEFAULT_NETPLAN//"{{IP}}"/"$IP"} +netplan=${netplan//"{{PREFIX}}"/"$PREFIX"} +netplan=${netplan//"{{GATEWAY}}"/"$GATEWAY"} +netplan=${netplan//"{{DNS}}"/"$DNS"} +echo -e "$netplan" > $BASE_PATH/network-config + +# Preparing image +if [ "$IMAGE_IS_URL" == true ]; then + curl -o $BASE_PATH/backing.img -L $IMAGE +fi + +qemu-img create -f qcow2 -F qcow2 \ + -o backing_file=$BASE_PATH/backing.img \ + $BASE_PATH/$NAME.qcow2 +qemu-img convert -f qcow2 -O raw $BASE_PATH/$NAME.qcow2 \ + $BASE_PATH/$NAME.img +qemu-img resize -f raw $BASE_PATH/$NAME.img $SIZE + +# Preparing cidata +genisoimage -output $BASE_PATH/cidata.iso -volid cidata -joliet \ + -rock $BASE_PATH/user-data $BASE_PATH/meta-data \ + $BASE_PATH/network-config + +# virt-install +virt-install --name $NAME \ + --os-variant $OS_VARIANT \ + --memory $MEMORY \ + --vcpus $VCPUS \ + --cpu host \ + --virt-type kvm \ + --disk path=$BASE_PATH/$NAME.img,format=raw,device=disk \ + --network bridge:$BRIDGE \ + --graphics vnc,listen=0.0.0.0,password=$VNC_PASSWORD \ + --import \ + --boot uefi \ + --disk $BASE_PATH/cidata.iso,device=cdrom \ + --noautoconsole \ + --noreboot + +# Attaching the drive for a test +virsh attach-disk $NAME \ + --source $SOURCE_DEV \ + --target $DEFAULT_DEV_INTERNAL_NAME \ + --cache none \ + --persistent + +# Starting vm +virsh start $NAME + +# Cleaning up existing known_hosts +cleanup_known_host $IP 2>/dev/null