From ede16d886335b4fa79f5e5a16ae55dc41446661e Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:33:03 +0200 Subject: [PATCH 1/6] Sync --- run/disk.sh | 98 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 27 deletions(-) diff --git a/run/disk.sh b/run/disk.sh index 8a0a146..129b620 100644 --- a/run/disk.sh +++ b/run/disk.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash set -eu +# Docker environment variabeles + +: ${DISK_IO:='native'} # I/O Mode, can be set to 'native', 'threads' or 'io_turing' +: ${DISK_ROTATION:='1'} # Rotation rate, set to 1 for SSD storage and increase for HDD +: ${DISK_CACHE:='none'} # Caching mode, can be set to 'writeback' for better performance + BOOT="$STORAGE/boot.img" [ ! -f "$BOOT" ] && echo "ERROR: Boot image does not exist ($BOOT)" && exit 81 @@ -15,20 +21,36 @@ if [ -f "${DATA}" ]; then if [ "$DATA_SIZE" -gt "$OLD_SIZE" ]; then echo "INFO: Resizing data disk from $OLD_SIZE to $DATA_SIZE bytes.." - - REQ=$((DATA_SIZE-OLD_SIZE)) - - # Check free diskspace - SPACE=$(df --output=avail -B 1 "${STORAGE}" | tail -n 1) - - if (( REQ > SPACE )); then - echo "ERROR: Not enough free space to resize virtual disk." && exit 84 - fi - if ! fallocate -l "${DATA_SIZE}" "${DATA}"; then - echo "ERROR: Could not allocate file for virtual disk." && exit 85 + if [ "$ALLOCATE" = "N" ]; then + + truncate -s "${DATA_SIZE}" "${DATA}"; + + else + + REQ=$((DATA_SIZE-OLD_SIZE)) + + # Check free diskspace + SPACE=$(df --output=avail -B 1 "${STORAGE}" | tail -n 1) + + if (( REQ > SPACE )); then + echo "ERROR: Not enough free space to resize data disk to ${DISK_SIZE}." + echo "ERROR: Specify a smaller size or disable preallocation with ALLOCATE=N." && exit 84 + fi + + if ! fallocate -l "${DATA_SIZE}" "${DATA}"; then + echo "ERROR: Could not allocate a file for the data disk." && exit 85 + fi + + if [ "$ALLOCATE" = "Z" ]; then + + GB=$(( (REQ + 1073741823)/1073741824 )) + + echo "INFO: Preallocating ${GB} GB of diskspace, please wait..." + dd if=/dev/urandom of="${DATA}" seek="${OLD_SIZE}" count="${REQ}" bs=1M iflag=count_bytes oflag=seek_bytes status=none + + fi fi - fi if [ "$DATA_SIZE" -lt "$OLD_SIZE" ]; then @@ -39,35 +61,57 @@ if [ -f "${DATA}" ]; then mv -f "${DATA}" "${DATA}.bak" fi - fi if [ ! -f "${DATA}" ]; then - # Check free diskspace - SPACE=$(df --output=avail -B 1 "${STORAGE}" | tail -n 1) - - if (( DATA_SIZE > SPACE )); then - echo "ERROR: Not enough free space to create virtual disk." && exit 86 - fi - # Create an empty file - if ! fallocate -l "${DATA_SIZE}" "${DATA}"; then - rm -f "${DATA}" - echo "ERROR: Could not allocate file for virtual disk." && exit 87 + + if [ "$ALLOCATE" = "N" ]; then + + truncate -s "${DATA_SIZE}" "${DATA}" + + else + + # Check free diskspace + SPACE=$(df --output=avail -B 1 "${STORAGE}" | tail -n 1) + + if (( DATA_SIZE > SPACE )); then + echo "ERROR: Not enough free space to create a data disk of ${DISK_SIZE}." + echo "ERROR: Specify a smaller size or disable preallocation with ALLOCATE=N." && exit 86 + fi + + if ! fallocate -l "${DATA_SIZE}" "${DATA}"; then + rm -f "${DATA}" + echo "ERROR: Could not allocate a file for the data disk." && exit 87 + fi + + if [ "$ALLOCATE" = "Z" ]; then + + echo "INFO: Preallocating ${DISK_SIZE} of diskspace, please wait..." + dd if=/dev/urandom of="${DATA}" count="${DATA_SIZE}" bs=1M iflag=count_bytes status=none + + fi fi # Check if file exists if [ ! -f "${DATA}" ]; then - echo "ERROR: Data image does not exist ($DATA)" && exit 88 + echo "ERROR: Data disk does not exist ($DATA)" && exit 88 fi fi -KVM_DISK_OPTS="\ +# Check the filesize +SIZE=$(stat -c%s "${DATA}") + +if [[ SIZE -ne DATA_SIZE ]]; then + echo "ERROR: Data disk has the wrong size: ${SIZE}" && exit 89 +fi + +DISK_OPTS="\ -drive id=cdrom0,if=none,format=raw,readonly=on,file=${BOOT} \ -device virtio-scsi-pci,id=scsi0 \ -device scsi-cd,bus=scsi0.0,drive=cdrom0 \ -device virtio-scsi-pci,id=hw-userdata,bus=pcie.0,addr=0xa \ - -drive file=${DATA},if=none,id=drive-userdata,format=raw,cache=none,aio=native,discard=on,detect-zeroes=on \ - -device scsi-hd,bus=hw-userdata.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata,id=userdata0,rotation_rate=1,bootindex=1" + -drive file=${DATA},if=none,id=drive-userdata,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=on,detect-zeroes=on \ + -device scsi-hd,bus=hw-userdata.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata,id=userdata0,rotation_rate=${DISK_ROTATION},bootindex=1" From 6feb39db28e9c5b6948af6fc9cea10e5fb25ffc1 Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:38:12 +0200 Subject: [PATCH 2/6] Sync --- run/network.sh | 149 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 120 insertions(+), 29 deletions(-) diff --git a/run/network.sh b/run/network.sh index c84638e..c8a7fc9 100644 --- a/run/network.sh +++ b/run/network.sh @@ -1,22 +1,87 @@ #!/usr/bin/env bash set -eu -: ${VM_NET_TAP:=''} -: ${VM_NET_IP:='20.20.20.21'} +# Docker environment variabeles + : ${VM_NET_HOST:='QEMU'} : ${VM_NET_MAC:='82:cf:d0:5e:57:66'} +: ${DHCP:='N'} : ${DNS_SERVERS:=''} -: ${DNSMASQ:='/usr/sbin/dnsmasq'} : ${DNSMASQ_OPTS:=''} +: ${DNSMASQ:='/usr/sbin/dnsmasq'} : ${DNSMASQ_CONF_DIR:='/etc/dnsmasq.d'} # ###################################### # Functions # ###################################### -# Setup macvtap device to connect later the VM and setup a new macvlan device to connect the host machine to the network -configureNatNetworks () { +configureDHCP() { + + # Create /dev/vhost-net + if [ ! -c /dev/vhost-net ]; then + mknod /dev/vhost-net c 10 238 + chmod 660 /dev/vhost-net + fi + + if [ ! -c /dev/vhost-net ]; then + echo -n "Error: VHOST interface not available. Please add the following " + echo "docker variable to your container: --device=/dev/vhost-net" && exit 85 + fi + + VM_NET_TAP="_VmMacvtap" + echo "Info: Retrieving IP via DHCP using MAC ${VM_NET_MAC}..." + + ip l add link eth0 name ${VM_NET_TAP} address ${VM_NET_MAC} type macvtap mode bridge || true + ip l set ${VM_NET_TAP} up + + ip a flush eth0 + ip a flush ${VM_NET_TAP} + + DHCP_IP=$( dhclient -v ${VM_NET_TAP} 2>&1 | grep ^bound | cut -d' ' -f3 ) + + if [[ "${DHCP_IP}" == [0-9.]* ]]; then + echo "Info: Retrieved IP ${DHCP_IP} via DHCP" + else + echo "ERROR: Cannot retrieve IP from DHCP using MAC ${VM_NET_MAC}" && exit 16 + fi + + ip a flush ${VM_NET_TAP} + + TAP_PATH="/dev/tap$(>$TAP_PATH; then + echo -n "ERROR: Please add the following docker variables to your container: " + echo "--device=/dev/vhost-net --device-cgroup-rule='c ${MAJOR}:* rwm'" && exit 21 + fi + + if ! exec 40>>/dev/vhost-net; then + echo -n "ERROR: VHOST can not be found. Please add the following docker " + echo "variable to your container: --device=/dev/vhost-net" && exit 22 + fi + + NET_OPTS="-netdev tap,id=hostnet0,vhost=on,vhostfd=40,fd=30" +} + +configureNAT () { + + VM_NET_IP='20.20.20.21' + VM_NET_TAP="_VmNatTap" #Create bridge with static IP for the VM guest brctl addbr dockerbridge @@ -32,6 +97,9 @@ configureNatNetworks () { iptables -t nat -A PREROUTING -i eth0 -p tcp -j DNAT --to $VM_NET_IP iptables -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to $VM_NET_IP + # Hack for guest VMs complaining about "bad udp checksums in 5 packets" + iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill + #Enable port forwarding flag [[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]] && sysctl -w net.ipv4.ip_forward=1 @@ -41,6 +109,33 @@ configureNatNetworks () { # Create lease file for faster resolve echo "0 $VM_NET_MAC $VM_NET_IP $VM_NET_HOST 01:${VM_NET_MAC}" > /var/lib/misc/dnsmasq.leases chmod 644 /var/lib/misc/dnsmasq.leases + + NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0" + + # Build DNS options from container /etc/resolv.conf + nameservers=($(grep '^nameserver' /etc/resolv.conf | sed 's/nameserver //')) + searchdomains=$(grep '^search' /etc/resolv.conf | sed 's/search //' | sed 's/ /,/g') + domainname=$(echo $searchdomains | awk -F"," '{print $1}') + + for nameserver in "${nameservers[@]}"; do + if ! [[ $nameserver =~ .*:.* ]]; then + [[ -z $DNS_SERVERS ]] && DNS_SERVERS=$nameserver || DNS_SERVERS="$DNS_SERVERS,$nameserver" + fi + done + + [[ -z $DNS_SERVERS ]] && DNS_SERVERS="1.1.1.1" + + DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:dns-server,$DNS_SERVERS --dhcp-option=option:router,${VM_NET_IP%.*}.1" + + if [ -n "$searchdomains" -a "$searchdomains" != "." ]; then + DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-search,$searchdomains --dhcp-option=option:domain-name,$domainname" + else + [[ -z $(hostname -d) ]] || DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-name,$(hostname -d)" + fi + + [ "$DEBUG" = "Y" ] && echo && echo "$DNSMASQ $DNSMASQ_OPTS" + + $DNSMASQ $DNSMASQ_OPTS } # ###################################### @@ -59,34 +154,30 @@ fi update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null -VM_NET_TAP="_VmNatTap" -configureNatNetworks -KVM_NET_OPTS="-netdev tap,ifname=${VM_NET_TAP},script=no,downscript=no,id=hostnet0" +GATEWAY=$(ip r | grep default | awk '{print $3}') -# Build DNS options from container /etc/resolv.conf -nameservers=($(grep '^nameserver' /etc/resolv.conf | sed 's/nameserver //')) -searchdomains=$(grep '^search' /etc/resolv.conf | sed 's/search //' | sed 's/ /,/g') -domainname=$(echo $searchdomains | awk -F"," '{print $1}') +if [ "$DEBUG" = "Y" ]; then -for nameserver in "${nameservers[@]}"; do - if ! [[ $nameserver =~ .*:.* ]]; then - [[ -z $DNS_SERVERS ]] && DNS_SERVERS=$nameserver || DNS_SERVERS="$DNS_SERVERS,$nameserver" - fi -done + IP=$(ip address show dev eth0 | grep inet | awk '/inet / { print $2 }' | cut -f1 -d/) + echo "Info: Container IP is ${IP} with gateway ${GATEWAY}" -[[ -z $DNS_SERVERS ]] && DNS_SERVERS="1.1.1.1" - -DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:dns-server,$DNS_SERVERS --dhcp-option=option:router,${VM_NET_IP%.*}.1" - -if [ -n "$searchdomains" -a "$searchdomains" != "." ]; then - DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-search,$searchdomains --dhcp-option=option:domain-name,$domainname" -else - [[ -z $(hostname -d) ]] || DNSMASQ_OPTS="$DNSMASQ_OPTS --dhcp-option=option:domain-name,$(hostname -d)" fi -$DNSMASQ $DNSMASQ_OPTS +if [ "$DHCP" != "Y" ]; then -KVM_NET_OPTS="${KVM_NET_OPTS} -device virtio-net-pci,romfile=,netdev=hostnet0,mac=${VM_NET_MAC},id=net0" + # Configuration for static IP + configureNAT -# Hack for guest VMs complaining about "bad udp checksums in 5 packets" -iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill +else + + if [[ "$GATEWAY" == "172."* ]]; then + echo -n "ERROR: You cannot enable DHCP while the container is " + echo "in a bridge network, only on a macvlan network!" && exit 86 + fi + + # Configuration for DHCP IP + configureDHCP + +fi + +NET_OPTS="${NET_OPTS} -device virtio-net-pci,romfile=,netdev=hostnet0,mac=${VM_NET_MAC},id=net0" From b012120489b693134c3a5dec5b1d0cb444da9e36 Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:40:31 +0200 Subject: [PATCH 3/6] Sync --- run/power.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/power.sh b/run/power.sh index 22fa8c9..260f8ee 100644 --- a/run/power.sh +++ b/run/power.sh @@ -56,4 +56,4 @@ _graceful_shutdown(){ _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT -KVM_MON_OPTS="-monitor telnet:localhost:${QEMU_MONPORT},server,nowait,nodelay" +MON_OPTS="-monitor telnet:localhost:${QEMU_MONPORT},server,nowait,nodelay" From 0730e94c1c788334d245beaa2aff0e293bed0072 Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:46:14 +0200 Subject: [PATCH 4/6] Sync --- run/run.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/run/run.sh b/run/run.sh index 0e36c5b..0257f23 100755 --- a/run/run.sh +++ b/run/run.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash set -eu +# Docker environment variabeles + +: ${URL:=''}. # URL of the ISO file +: ${DEBUG:=''}. # Enable debug mode +: ${ALLOCATE:='Y'} # Preallocate diskspace +: ${CPU_CORES:='1'} # vCPU count +: ${DISK_SIZE:='16G'} # Initial disk size +: ${RAM_SIZE:='512M'} # Amount of RAM + echo "Starting QEMU for Docker v${VERSION}..." STORAGE="/storage" @@ -20,20 +29,26 @@ fi # Configure shutdown . /run/power.sh -KVM_ACC_OPTS="" +KVM_OPTS="" if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then if [[ $(grep -e vmx -e svm /proc/cpuinfo) ]]; then - KVM_ACC_OPTS="-machine type=q35,usb=off,accel=kvm -enable-kvm -cpu host" + KVM_OPTS=",accel=kvm -enable-kvm -cpu host" fi fi -[ -z "${KVM_ACC_OPTS}" ] && echo "Error: KVM acceleration is disabled.." && exit 88 +if [ -z "${KVM_OPTS}" ]; then + echo "Error: KVM acceleration is disabled.." + [ "$DEBUG" != "Y" ] && exit 88 +fi -RAM_SIZE=$(echo "${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') -KVM_SERIAL_OPTS="-serial mon:stdio -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x3" -EXTRA_OPTS="-nographic -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0 -device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4" -ARGS="-m ${RAM_SIZE} -smp ${CPU_CORES} ${KVM_ACC_OPTS} ${EXTRA_OPTS} ${KVM_MON_OPTS} ${KVM_SERIAL_OPTS} ${KVM_NET_OPTS} ${KVM_DISK_OPTS}" +DEF_OPTS="-nographic -nodefaults" +KVM_OPTS="-machine type=q35,usb=off${KVM_OPTS}" +RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') +CPU_OPTS="-smp ${CPU_CORES},sockets=1,cores=${CPU_CORES},threads=1" +SERIAL_OPTS="-serial mon:stdio -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x3" +EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0 -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0" +ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${KVM_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS}" set -m ( From 494d225e2eeb152ba3eada794e358e218e412b61 Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:53:37 +0200 Subject: [PATCH 5/6] Sync --- Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 049ad6b..7677ff5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,14 @@ FROM debian:bookworm-20230411-slim RUN apt-get update && apt-get -y upgrade && \ apt-get --no-install-recommends -y install \ wget \ - dnsmasq \ iptables \ iproute2 \ + dnsmasq \ + net-tools \ bridge-utils \ - netcat-openbsd \ ca-certificates \ + isc-dhcp-client \ + netcat-openbsd \ qemu-system-x86 \ && apt-get clean @@ -19,15 +21,15 @@ VOLUME /storage EXPOSE 22 -ENV CPU_CORES 1 -ENV DISK_SIZE 16G -ENV RAM_SIZE 512M +ENV ALLOCATE "Y" +ENV CPU_CORES "1" +ENV DISK_SIZE "16G" +ENV RAM_SIZE "512M" +ENV BOOT "http://www.example.com/image.iso" ARG BUILD_ARG=0 ARG VERSION_ARG="0.0" ENV BUILD=$BUILD_ARG ENV VERSION=$VERSION_ARG -ENV BOOT http://www.example.com/image.iso - ENTRYPOINT ["/run/run.sh"] From fe41e1c28efb7d78e7fc65ede883609e18fd2ad6 Mon Sep 17 00:00:00 2001 From: Kroese Date: Thu, 20 Apr 2023 18:54:02 +0200 Subject: [PATCH 6/6] Boot --- run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/run.sh b/run/run.sh index 0257f23..6b950ce 100755 --- a/run/run.sh +++ b/run/run.sh @@ -3,7 +3,7 @@ set -eu # Docker environment variabeles -: ${URL:=''}. # URL of the ISO file +: ${BOOT:=''}. # URL of the ISO file : ${DEBUG:=''}. # Enable debug mode : ${ALLOCATE:='Y'} # Preallocate diskspace : ${CPU_CORES:='1'} # vCPU count