diff --git a/readme.md b/readme.md index b50daf1..34c924a 100644 --- a/readme.md +++ b/readme.md @@ -17,7 +17,6 @@ Virtual DSM in a docker container. - Multi-platform - KVM acceleration - GPU passthrough - - Graceful shutdowns - Upgrades supported ## Usage @@ -85,6 +84,32 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti ``` Keep in mind that this will not affect any of your existing disks, it only applies to newly created disks. + + * ### How do I add multiple disks? + + To mount extra volumes, modify your compose file like this: + + ```yaml + environment: + DISK2_SIZE: "32G" + DISK3_SIZE: "64G" + volumes: + - /home/example2:/storage2 + - /home/example3:/storage3 + ``` + + Additionally, it's also possible to passthrough raw disk devices like this: + + ```yaml + environment: + DEVICE2: "/dev/vdc1" + DEVICE3: "/dev/vdc2" + devices: + - /dev/vdc1 + - /dev/vdc2 + ``` + + Please beware that any pre-existing partitions and data on those devices will be wiped. * ### How do I increase the amount of CPU or RAM? diff --git a/run/check.sh b/run/check.sh index 3262087..8fdab35 100644 --- a/run/check.sh +++ b/run/check.sh @@ -2,49 +2,60 @@ set -u [ ! -f "/run/qemu.pid" ] && echo "QEMU not running yet.." && exit 0 -[ -f "/run/qemu.counter" ] && echo "QEMU is shutting down.." && exit 1 +[ -f "/run/qemu.count" ] && echo "QEMU is shutting down.." && exit 1 -# Retrieve IP from guest VM for Docker healthcheck -RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1) +file="/run/dsm.url" + +if [ ! -f "$file" ]; then + + # Retrieve IP from guest VM for Docker healthcheck + RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1) + + if [[ ! "${RESPONSE}" =~ "\"success\"" ]] ; then + echo "Failed to connect to guest: $RESPONSE" && exit 1 + fi + + # Retrieve the HTTP port number + if [[ ! "${RESPONSE}" =~ "\"http_port\"" ]] ; then + echo "Failed to parse response from guest: $RESPONSE" && exit 1 + fi + + rest=${RESPONSE#*http_port} + rest=${rest#*:} + rest=${rest%%,*} + PORT=${rest%%\"*} + + [ -z "${PORT}" ] && echo "Guest has not set a portnumber yet.." && exit 1 + + # Retrieve the IP address + if [[ ! "${RESPONSE}" =~ "eth0" ]] ; then + echo "Failed to parse response from guest: $RESPONSE" && exit 1 + fi + + rest=${RESPONSE#*eth0} + rest=${rest#*ip} + rest=${rest#*:} + rest=${rest#*\"} + IP=${rest%%\"*} + + [ -z "${IP}" ] && echo "Guest has not received an IP yet.." && exit 1 + + echo "${IP}:${PORT}" > $file -if [[ ! "${RESPONSE}" =~ "\"success\"" ]] ; then - echo "Failed to connect to guest: $RESPONSE" && exit 1 fi -# Retrieve the HTTP port number -if [[ ! "${RESPONSE}" =~ "\"http_port\"" ]] ; then - echo "Failed to parse response from guest: $RESPONSE" && exit 1 -fi +LOCATION=$(cat "$file") -rest=${RESPONSE#*http_port} -rest=${rest#*:} -rest=${rest%%,*} -PORT=${rest%%\"*} - -[ -z "${PORT}" ] && echo "Guest has not set a portnumber yet.." && exit 1 - -# Retrieve the IP address -if [[ ! "${RESPONSE}" =~ "eth0" ]] ; then - echo "Failed to parse response from guest: $RESPONSE" && exit 1 -fi - -rest=${RESPONSE#*eth0} -rest=${rest#*ip} -rest=${rest#*:} -rest=${rest#*\"} -IP=${rest%%\"*} - -[ -z "${IP}" ] && echo "Guest has not received an IP yet.." && exit 1 - -if ! curl -m 3 -ILfSs "http://${IP}:${PORT}/" > /dev/null; then - echo "Failed to reach ${IP}:${PORT}" +if ! curl -m 20 -ILfSs "http://${LOCATION}/" > /dev/null; then + rm -f $file + echo "Failed to reach http://${LOCATION}" exit 1 fi -if [[ "$IP" == "20.20"* ]]; then +if [[ "$LOCATION" == "20.20"* ]]; then echo "Healthcheck OK" else - echo "Healthcheck OK ( $IP )" + echo "Healthcheck OK ( ${LOCATION%:*} )" fi exit 0 diff --git a/run/config.sh b/run/config.sh new file mode 100644 index 0000000..1805072 --- /dev/null +++ b/run/config.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -Eeuo pipefail + +KVM_ERR="" +KVM_OPTS="" + +if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then + if ! grep -q -e vmx -e svm /proc/cpuinfo; then + KVM_ERR="(vmx/svm disabled)" + fi +else + [ -e /dev/kvm ] && KVM_ERR="(no write access)" || KVM_ERR="(device file missing)" +fi + +if [ -n "${KVM_ERR}" ]; then + if [ "$ARCH" == "amd64" ]; then + error "KVM acceleration not detected ${KVM_ERR}, see the FAQ about this." + [[ "${DEBUG}" != [Yy1]* ]] && exit 88 + fi +else + KVM_OPTS=",accel=kvm -enable-kvm -cpu host" +fi + +DEF_OPTS="-nographic -nodefaults -boot strict=on -display none" +RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') +CPU_OPTS="-smp ${CPU_CORES},sockets=1,dies=1,cores=${CPU_CORES},threads=1" +MAC_OPTS="-machine type=q35,usb=off,dump-guest-core=off,hpet=off${KVM_OPTS}" +EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4" +EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom" +EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c" + +if [[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]]; then + DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128" + DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1" +fi + +ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}" +ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ') + +return 0 diff --git a/run/disk.sh b/run/disk.sh index 23794da..b1e49d1 100644 --- a/run/disk.sh +++ b/run/disk.sh @@ -237,3 +237,5 @@ if [ -n "${DEVICE3}" ]; then -device scsi-hd,bus=hw-userdata6.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata6,id=userdata6,rotation_rate=${DISK_ROTATION},bootindex=8" fi + +return 0 diff --git a/run/gpu.sh b/run/gpu.sh index b72f48b..fab3e2d 100644 --- a/run/gpu.sh +++ b/run/gpu.sh @@ -1,6 +1,10 @@ #!/bin/bash set -Eeuo pipefail +if [[ "${GPU}" != [Yy1]* ]] || [[ "$ARCH" != "amd64" ]]; then + return 0 +fi + [ ! -d /dev/dri ] && mkdir -m 755 /dev/dri if [ ! -c /dev/dri/card0 ]; then @@ -14,9 +18,6 @@ fi chmod 666 /dev/dri/card0 chmod 666 /dev/dri/renderD128 -DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128" -DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1" - if ! apt-mark showinstall | grep -q "xserver-xorg-video-intel"; then info "Installing Intel GPU drivers..." @@ -40,3 +41,5 @@ if ! apt-mark showinstall | grep -q "qemu-system-modules-opengl"; then apt-get -qq --no-install-recommends -y install qemu-system-modules-opengl > /dev/null fi + +return 0 diff --git a/run/install.sh b/run/install.sh index 1690cbf..33b93b0 100644 --- a/run/install.sh +++ b/run/install.sh @@ -1,6 +1,23 @@ #!/usr/bin/env bash set -Eeuo pipefail +STORAGE="/storage" +[ ! -d "$STORAGE" ] && error "Storage folder (${STORAGE}) not found!" && exit 13 + +if [ -f "$STORAGE"/dsm.ver ]; then + BASE=$(cat "${STORAGE}/dsm.ver") +else + # Fallback for old installs + BASE="DSM_VirtualDSM_42962" +fi + +[ -n "$URL" ] && BASE=$(basename "$URL" .pat) + +if [[ -f "$STORAGE/$BASE.boot.img" ]] && [[ -f "$STORAGE/$BASE.system.img" ]]; then + # Previous installation found + return 0 +fi + # Display wait message /run/server.sh 5000 install & @@ -31,7 +48,6 @@ rm -f "$STORAGE"/"$BASE".agent rm -f "$STORAGE"/"$BASE".boot.img rm -f "$STORAGE"/"$BASE".system.img -TMP="/tmp/dsm" MIN_SPACE=6442450944 FS=$(stat -f -c %T "$STORAGE") @@ -39,10 +55,13 @@ if [[ "$FS" != "fat"* && "$FS" != "vfat"* && "$FS" != "exfat"* && \ "$FS" != "ntfs"* && "$FS" != "fuse"* && "$FS" != "msdos"* ]]; then TMP="$STORAGE/tmp" else + TMP="/tmp/dsm" SPACE=$(df --output=avail -B 1 /tmp | tail -n 1) (( MIN_SPACE > SPACE )) && TMP="$STORAGE/tmp" fi +rm -rf /tmp/dsm +rm -rf "$STORAGE/tmp" rm -rf "$TMP" && mkdir -p "$TMP" # Check free diskspace diff --git a/run/power.sh b/run/power.sh index cfd107e..eda20bd 100644 --- a/run/power.sh +++ b/run/power.sh @@ -3,14 +3,14 @@ set -Eeuo pipefail # Configure QEMU for graceful shutdown -QEMU_MONPORT=7100 -QEMU_POWERDOWN_TIMEOUT=50 +QEMU_PORT=7100 +QEMU_TIMEOUT=50 -_QEMU_PID=/run/qemu.pid -_QEMU_SHUTDOWN_COUNTER=/run/qemu.counter +QEMU_PID=/run/qemu.pid +QEMU_COUNT=/run/qemu.count -rm -f "${_QEMU_PID}" -rm -f "${_QEMU_SHUTDOWN_COUNTER}" +rm -f "${QEMU_PID}" +rm -f "${QEMU_COUNT}" _trap(){ func="$1" ; shift @@ -23,14 +23,14 @@ _graceful_shutdown() { set +e - [ ! -f "${_QEMU_PID}" ] && return - [ -f "${_QEMU_SHUTDOWN_COUNTER}" ] && return + [ ! -f "${QEMU_PID}" ] && exit 130 + [ -f "${QEMU_COUNT}" ] && return echo && info "Received $1 signal, shutting down..." - echo 0 > "${_QEMU_SHUTDOWN_COUNTER}" + echo 0 > "${QEMU_COUNT}" # Don't send the powerdown signal because vDSM ignores ACPI signals - # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" > /dev/null + # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_PORT}" > /dev/null # Send shutdown command to guest agent via serial port RESPONSE=$(curl -s -m 5 -S http://127.0.0.1:2210/read?command=6 2>&1) @@ -39,22 +39,22 @@ _graceful_shutdown() { echo && error "Could not send shutdown command to the guest ($RESPONSE)" - kill -15 "$(cat "${_QEMU_PID}")" + kill -15 "$(cat "${QEMU_PID}")" pkill -f qemu-system-x86_64 || true fi - while [ "$(cat ${_QEMU_SHUTDOWN_COUNTER})" -lt "${QEMU_POWERDOWN_TIMEOUT}" ]; do + while [ "$(cat ${QEMU_COUNT})" -lt "${QEMU_TIMEOUT}" ]; do # Increase the counter - echo $(($(cat ${_QEMU_SHUTDOWN_COUNTER})+1)) > ${_QEMU_SHUTDOWN_COUNTER} + echo $(($(cat ${QEMU_COUNT})+1)) > ${QEMU_COUNT} # Try to connect to qemu - if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 ; then + if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 ; then sleep 1 - CNT="$(cat ${_QEMU_SHUTDOWN_COUNTER})/${QEMU_POWERDOWN_TIMEOUT}" + CNT="$(cat ${QEMU_COUNT})/${QEMU_TIMEOUT}" [[ "${DEBUG}" == [Yy1]* ]] && info "Shutting down, waiting... (${CNT})" fi @@ -62,7 +62,7 @@ _graceful_shutdown() { done echo && echo "❯ Quitting..." - echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 || true + echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 || true closeNetwork @@ -71,4 +71,4 @@ _graceful_shutdown() { _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT -MON_OPTS="-monitor telnet:localhost:${QEMU_MONPORT},server,nowait,nodelay" +MON_OPTS="-monitor telnet:localhost:${QEMU_PORT},server,nowait,nodelay" diff --git a/run/print.sh b/run/print.sh index 93a47cd..5049d95 100644 --- a/run/print.sh +++ b/run/print.sh @@ -4,12 +4,13 @@ set -Eeuo pipefail info () { echo -e >&2 "\E[1;34m❯\E[1;36m $1\E[0m" ; } error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; } -retry=true +file="/run/dsm.url" -while [ "$retry" = true ] +while [ ! -f "$file" ] do sleep 3 + [ -f "$file" ] && continue # Retrieve IP from guest VM @@ -46,14 +47,16 @@ do [ -z "${IP}" ] && continue - retry=false + echo "${IP}:${PORT}" > $file done -if [[ "$IP" == "20.20"* ]]; then - MSG="port ${PORT}" +LOCATION=$(cat "$file") + +if [[ "$LOCATION" == "20.20"* ]]; then + MSG="port ${LOCATION##*:}" else - MSG="http://${IP}:${PORT}" + MSG="http://${LOCATION}" fi echo "" >&2 diff --git a/run/reset.sh b/run/reset.sh new file mode 100644 index 0000000..f7c2543 --- /dev/null +++ b/run/reset.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +info () { echo -e "\E[1;34m❯ \E[1;36m$1\E[0m" ; } +error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; } +trap 'error "Status $? while: ${BASH_COMMAND} (line $LINENO/$BASH_LINENO)"' ERR + +[ ! -f "/run/run.sh" ] && error "Script must run inside Docker container!" && exit 11 +[ "$(id -u)" -ne "0" ] && error "Script must be executed with root privileges." && exit 12 + +# Docker environment variables + +: ${URL:=''} # URL of the PAT file +: ${GPU:='N'} # Enable GPU passthrough +: ${DEBUG:='N'} # Enable debugging mode +: ${ALLOCATE:='Y'} # Preallocate diskspace +: ${ARGUMENTS:=''} # Extra QEMU parameters +: ${CPU_CORES:='1'} # Amount of CPU cores +: ${DISK_SIZE:='16G'} # Initial data disk size +: ${RAM_SIZE:='512M'} # Maximum RAM amount + +# Helper variables + +KERNEL=$(uname -r | cut -b 1) +MINOR=$(uname -r | cut -d '.' -f2) +ARCH=$(dpkg --print-architecture) +VERS=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1) + +# Cleanup files + +rm -f /run/dsm.url +rm -f /run/qemu.pid +rm -f /run/qemu.count + +return 0 diff --git a/run/run.sh b/run/run.sh index 7094fec..fbb19a7 100755 --- a/run/run.sh +++ b/run/run.sh @@ -1,97 +1,26 @@ #!/usr/bin/env bash set -Eeuo pipefail -# Docker environment variables - -: ${URL:=''} # URL of the PAT file -: ${GPU:='N'} # Enable GPU passthrough -: ${DEBUG:='N'} # Enable debugging mode -: ${ALLOCATE:='Y'} # Preallocate diskspace -: ${ARGUMENTS:=''} # Extra QEMU parameters -: ${CPU_CORES:='1'} # Amount of CPU cores -: ${DISK_SIZE:='16G'} # Initial data disk size -: ${RAM_SIZE:='512M'} # Maximum RAM amount - echo "❯ Starting Virtual DSM for Docker v${VERSION}..." +echo "❯ For support visit https://github.com/vdsm/virtual-dsm/" -info () { echo -e "\E[1;34m❯ \E[1;36m$1\E[0m" ; } -error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; } -trap 'error "Status $? while: ${BASH_COMMAND} (line $LINENO/$BASH_LINENO)"' ERR - -[ ! -f "/run/run.sh" ] && error "Script must run inside Docker container!" && exit 11 -[ "$(id -u)" -ne "0" ] && error "Script must be executed with root privileges." && exit 12 - -STORAGE="/storage" -KERNEL=$(uname -r | cut -b 1) -MINOR=$(uname -r | cut -d '.' -f2) -ARCH=$(dpkg --print-architecture) -VERS=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1) - -[ ! -d "$STORAGE" ] && error "Storage folder (${STORAGE}) not found!" && exit 13 - -if [ -f "$STORAGE"/dsm.ver ]; then - BASE=$(cat "${STORAGE}/dsm.ver") -else - # Fallback for old installs - BASE="DSM_VirtualDSM_42962" -fi - -[ -n "$URL" ] && BASE=$(basename "$URL" .pat) - -if [[ ! -f "$STORAGE/$BASE.boot.img" ]] || [[ ! -f "$STORAGE/$BASE.system.img" ]]; then - . /run/install.sh -fi - +. /run/reset.sh # Initialize system +. /run/install.sh # Run installation . /run/disk.sh # Initialize disks . /run/network.sh # Initialize network +. /run/gpu.sh # Initialize graphics . /run/serial.sh # Initialize serialport . /run/power.sh # Configure shutdown - -KVM_ERR="" -KVM_OPTS="" - -if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then - if ! grep -q -e vmx -e svm /proc/cpuinfo; then - KVM_ERR="(vmx/svm disabled)" - fi -else - [ -e /dev/kvm ] && KVM_ERR="(no write access)" || KVM_ERR="(device file missing)" -fi - -if [ -n "${KVM_ERR}" ]; then - if [ "$ARCH" == "amd64" ]; then - error "KVM acceleration not detected ${KVM_ERR}, see the FAQ about this." - [[ "${DEBUG}" != [Yy1]* ]] && exit 88 - fi -else - KVM_OPTS=",accel=kvm -enable-kvm -cpu host" -fi - -DEF_OPTS="-nographic -nodefaults -boot strict=on -display none" -RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g') -CPU_OPTS="-smp ${CPU_CORES},sockets=1,dies=1,cores=${CPU_CORES},threads=1" -MAC_OPTS="-machine type=q35,usb=off,dump-guest-core=off,hpet=off${KVM_OPTS}" -EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4" -EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom" -EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c" - -[[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]] && . /run/gpu.sh - -ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}" -ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ') +. /run/config.sh # Configure arguments trap - ERR set -m ( [[ "${DEBUG}" == [Yy1]* ]] && info "$VERS" && set -x - qemu-system-x86_64 ${ARGS:+ $ARGS} & echo $! > "${_QEMU_PID}" + qemu-system-x86_64 ${ARGS:+ $ARGS} & echo $! > "${QEMU_PID}" { set +x; } 2>/dev/null ) set +m -#if (( KERNEL > 5 )) || ( (( KERNEL == 5 )) && (( MINOR > 2 )) ); then -# pidwait -F "${_QEMU_PID}" & wait $! -#else - -tail --pid "$(cat "${_QEMU_PID}")" --follow /dev/null & wait $! +tail --pid "$(cat "${QEMU_PID}")" --follow /dev/null & wait $! diff --git a/run/serial.sh b/run/serial.sh index 02fdb3a..3df1676 100644 --- a/run/serial.sh +++ b/run/serial.sh @@ -51,3 +51,5 @@ SERIAL_OPTS="\ -device isa-serial,chardev=charserial0,id=serial0 \ -chardev socket,id=charchannel0,host=127.0.0.1,port=12345,reconnect=10 \ -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=vchannel" + +return 0 diff --git a/run/server.sh b/run/server.sh index c592914..4f5f91a 100644 --- a/run/server.sh +++ b/run/server.sh @@ -46,17 +46,15 @@ if [[ "$2" != "/run/ip.sh" ]]; then else - BODY="The location of DSM is http://\${IP}:\${PORT}" + BODY="The location of DSM is http://\${LOCATION}" WAIT="Please wait while discovering IP..." HTML=$(html "xxx") { echo "#!/bin/bash" - echo "INFO=\$(curl -s -m 2 -S http://127.0.0.1:2210/read?command=10 2>/dev/null)" - echo "rest=\${INFO#*http_port}; rest=\${rest#*:}; rest=\${rest%%,*}; PORT=\${rest%%\\\"*}" - echo "rest=\${INFO#*eth0}; rest=\${rest#*ip}; rest=\${rest#*:}; rest=\${rest#*\\\"}; IP=\${rest%%\\\"*}" - echo "HTML=\"$HTML\"; [ -z \"\${IP}\" ] && BODY=\"$WAIT\" || BODY=\"$BODY\"; HTML=\${HTML/xxx/\$BODY}" + echo "[ -f \"/run/dsm.url\" ] && LOCATION=\$(cat \"/run/dsm.url\")" + echo "HTML=\"$HTML\"; [ -z \"\${LOCATION}\" ] && BODY=\"$WAIT\" || BODY=\"$BODY\"; HTML=\${HTML/xxx/\$BODY}" echo "printf '%b' \"HTTP/1.1 200 OK\\nContent-Length: \${#HTML}\\nConnection: close\\n\\n\$HTML\"" } > "$TMP_FILE"