#!/usr/local/bin/cbsd
#v11.1.17
MYARG=""
MYOPTARG="jname inter debug checkpoint"
MYDESC="Start jail"
ADDHELP="inter=0 to prevent any questions and to accept answers by default\n\
  checkpoint - start from specified checkpoint name\n"
CBSDMODULE="bhyve"
EXTHELP="wf_jstop_jstart.html"

. ${subr}
. ${system}
. ${strings}
. ${workdir}/universe.subr
. ${workdir}/bhyve.subr
. ${tools}
. ${workdir}/vnet.subr # get_vm_uplink_interface

readconf buildworld.conf
readconf jail-freebsd-default.conf

[ -z "${1}" ] && select_jail_by_list -s "List of offline VMs" -a "Off" -e bls -r ${sqlreplica}
init $*

. ${workdir}/fetch.subr
. ${workdir}/jcreate.subr	# for external_exec_master_script
. ${workdir}/virtual.subr	# for init_systap

create_from_jail()
{
	local passt_args=

	[ -n "${pci_passthru_args}" ] && passt_args="-S"

	# from jail2iso mode
	case "${vm_efi}" in
		uefi_csm|uefi)
			bhyveload_cmd=""
		;;
		*)
			bhyveload_cmd="/usr/bin/lockf -s -t0 /tmp/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
		;;
	esac
}

start_bhyve()
{
	local bhyveload_cmd _err
	local passt_args=

	local is_ppt=$( cbsdsql local SELECT ppt FROM bhyveppt WHERE jname=\"${jname}\" 2>/dev/null )
	[ -n "${is_ppt}" ] && passt_args="-S"

	# flags which means that the CD as boot device will use only once
	cd_boot_once=0

	# profile
	readconf vm-${vm_os_type}-${vm_os_profile}.conf
	if [ -z "${vm_profile}" ]; then
		${ECHO} "${MAGENTA}No such profile: ${GREEN}vm-${vm_os_type}-${vm_os_profile}.conf${NORMAL}"
		sleep 2
	fi
	# re-read jail params and apply personal after profile
	. ${jrcconf}

	# CBSD QUEUE
	if [ -x "${moduledir}/cbsd_queue.d/cbsd_queue" ]; then
		[ "${cbsd_queue_name}" != "none" ] && cbsd_queue cbsd_queue_name=${cbsd_queue_name} id=${jname} cmd=bstart status=1
	fi

	vm_boot=$( cbsdsql local SELECT vm_boot FROM bhyve WHERE jname=\"${jname}\" 2>/dev/null )

	case "${vm_os_type}" in
		freebsd)
			if [ "${from_jail}" = "1" ]; then
				create_from_jail
			else
				# default mode
				check_for_empty_hdd
				if [ "${vm_boot}" = "cd" ]; then
					init_iso
					if [ $? -eq 1 ]; then
						printf "${MAGENTA}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${NORMAL}"
						for i in $( /usr/bin/jot 5 ); do
							printf "."
							sleep 1
						done
						echo
					fi
				fi

				case "${vm_efi}" in
					uefi_csm|uefi)
						bhyveload_cmd=""
						;;
					*)
						if [ "${vm_boot}" = "cd" ]; then
							bhyveload_cmd="/usr/bin/lockf -s -t0 /tmp/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -d ${iso_img} -m ${vm_ram} ${jname}"
							bhyveload_cmd_once="/usr/bin/lockf -s -t0 /tmp/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
						else
							bhyveload_cmd="/usr/bin/lockf -s -t0 /tmp/bhyveload.${jname}.lock /usr/sbin/bhyveload ${passt_args} -m ${vm_ram} -d ${data}/${MDFILE} ${jname}"
						fi
						;;
				esac
			fi
			;;
		linux|other|windows)
			# profile
			readconf vm-${vm_os_type}-${vm_os_profile}.conf

			[ -z "${vm_profile}" ] && err 1 "${MAGENTA}No such profile: ${GREEN}vm-${vm_os_type}-${vm_os_profile}.conf${NORMAL}"
			# re-read jail params and apply personal after profile 
			. ${jrcconf}
			check_for_empty_hdd

			if [ "${vm_boot}" = "cd" ]; then
				init_iso
				if [ $? -eq 1 ]; then
					printf "${MAGENTA}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${NORMAL}"
					for i in $( /usr/bin/jot 5 ); do
						printf "."
						sleep 1
					done
					echo
				fi
			fi
			bhyveload_cmd=""
			;;

		openbsd|dflybsd|netbsd)
			# profile
			readconf vm-${vm_os_type}-${vm_os_profile}.conf
			[ -z "${vm_profile}" ] && err 1 "${MAGENTA}No such profile: ${GREEN}vm-${vm_os_type}-${vm_os_profile}.conf${NORMAL}"
			# re-read jail params and apply personal after profile
			. ${jrcconf}
			check_for_empty_hdd
			if [ "${vm_boot}" = "cd" ]; then
				init_iso
				if [ $? -eq 1 ]; then
					printf "${MAGENTA}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${NORMAL}"
					for i in $( /usr/bin/jot 5 ); do
						printf "."
						sleep 1
					done
					echo
				fi
			fi

			_err=$( /usr/bin/file -s ${data}/${MDFILE}| /usr/bin/cut -d":" -f2| /usr/bin/xargs )

			if [ "${_err}" = "data" -a "${vm_boot}" = "hdd" ]; then
				${ECHO} "${MAGENTA}Looks like ${GREEN}${data}/${MDFILE}${MAGENTA} is empty.${NORMAL}"
				if getyesno "May be you want to boot from CD?"; then
					vm_boot="cd";
				fi
			fi

			bhyveload_cmd=""
			;;
		*)
			err 1 "${MAGENTA}Unknown vm profile: ${GREEN}${vm_os_type}${NORMAL}"
	esac

	# for vnet we can make another action
	. ${vimageconf}

	#unset zero-value
	[ "${bhyve_flags}" = "0" ] && unset bhyve_flags
	[ "${vm_os_profile}" = "0" ] && unset vm_os_profile

	# init bhyve_cpus
	if ! compile_bhyve_cpus_args; then
		${ECHO} "${MAGENTA}Unable to compile bhyve_cpus_args for VMs: ${GREEN}${jname}${NORMAL}"
		bhyve_cpus="${vm_cpus}"
	fi

	# init hostbridge_args
	if ! compile_hostbridge_args; then
		${ECHO} "${MAGENTA}No such hostbridge for VMs: ${GREEN}${jname}${NORMAL}"
		unset hostbridge_args
	fi

	# Must be after hoster and before cd/hdd
	if ! compile_uefi_boot_args; then
		unset uefi_boot_args
	fi

	# Must be after hoster bridge to take 2-4 slot id
	case "${vm_boot}" in
		hdd|net)
			# init dsk_args first
			if ! compile_dsk_args; then
				${ECHO} "${MAGENTA}No such disk for VMs: ${GREEN}${jname}${NORMAL}"
				unset dsk_args
			fi

			# init cd_args
			if ! compile_cd_args; then
				unset cd_args
			fi
			;;
		cd)
			# init cd_args first
			if ! compile_cd_args; then
				unset cd_args
			fi

			# init dsk_args
			if ! compile_dsk_args; then
				${ECHO} "${MAGENTA}No such disk for VMs: ${GREEN}${jname}${NORMAL}"
				unset dsk_args
			fi
			;;
		*)
			err 1 "${MAGENTA}Unknown vm_boot method: ${GREEN}${vm_boot}${NORMAL}"
			;;
	esac

	if ! compile_dsk_controller_args; then
		unset dsk_controller_args
	fi

	# init nic_args
	if ! compile_nic_args ; then
		${ECHO} "${MAGENTA}No such nic for VMs: ${GREEN}${jname}${NORMAL}"
		unset nic_args
	fi

	# init pci_passthru_args
	if ! compile_pci_passthru_args; then
		${ECHO} "${MAGENTA}No such pci_passthru for VMs: ${GREEN}${jname}${NORMAL}"
		unset pci_passthru_args
	fi

	# init console_args
	if ! compile_console_args; then
		${ECHO} "${MAGENTA}No such console for VMs: ${GREEN}${jname}${NORMAL}"
		unset console_args
	fi

	# init virtiornd_args
	if ! compile_virtiornd_args; then
		${ECHO} "${MAGENTA}No such rnd for VMs: ${GREEN}${jname}${NORMAL}"
		unset lpc_args
	fi

	# init VirtFS compile_virtio_9p_args
	if ! compile_virtio_9p_args; then
		${ECHO} "${MAGENTA}No such 9p for VMs: ${GREEN}${jname}${NORMAL}"
		unset virtio_9p_args
	fi

	# init efi_args
	if ! compile_efi_args; then
		${ECHO} "${MAGENTA}No such efi for VMs: ${GREEN}${jname}${NORMAL}"
		unset efi_args
	fi

	# init lpc_args
	if ! compile_lpc_args; then
		${ECHO} "${MAGENTA}No such lpc for VMs: ${GREEN}${jname}${NORMAL}"
		unset lpc_args
	fi


	# init vnc_args
	if ! compile_vnc_args ; then
		unset vnc_args
	fi

	# Poehali!
	passthr=""

	# for init external hook variables
	geniplist ${ip4_addr}

	export_bhyve_data_for_external_hook
	external_exec_master_script "master_prestart.d"

	vm_logfile=$( /usr/bin/mktemp )

	# The parameters in local rc.conf file can overwrite the settings in the database
	[ -r ${jailsysdir}/${jname}/etc/rc.conf ] && . ${jailsysdir}/${jname}/etc/rc.conf

	[ -r "${jailsysdir}/${jname}/helpers/brctl.sqlite" ] && nice=$( cbsdsql ${jailsysdir}/${jname}/helpers/brctl.sqlite "SELECT cur FROM forms WHERE param=\"nice\"" )
	[ -z "${nice}" ] && nice="0"
	[ "${nice}" != "0" ] && ${ECHO} "${MAGENTA}bhyve renice: ${GREEN}${nice}${NORMAL}"

/bin/cat > ${jailsysdir}/${jname}/bhyve.conf <<EOF
vm_boot='${vm_boot}'
bhyveload_cmd='${bhyveload_cmd}'
tmuxcmd='${tmuxcmd}'
jname='${jname}'

bhyve_flags='${bhyve_flags}'
# with topology
vm_cpus='${bhyve_cpus}'
vm_ram='${vm_ram}'
vm_efi='${vm_efi}'
vm_console='${vm_console}'
hostbridge_args='${hostbridge_args}'
passthr='${passthr}'
lpc_args='${lpc_args}'
pci_passthru_args='${pci_passthru_args}'
virtiornd_args='${virtiornd_args}'
nic_args='${nic_args}'
uefi_boot_args='${uefi_boot_args}'
dsk_args='${dsk_args}'
dsk_controller_args='${dsk_controller_args}'
cd_args='${cd_args}'
cd_args2='${cd_args2}'
#
efi_args='${efi_args}'
vnc_args='${vnc_args}'
console_args='${console_args}'
mytap='${mytap}'
cd_boot_once='${cd_boot_once}'
bhyveload_cmd_once='${bhyveload_cmd_once}'
console_nmdm='${console_nmdm}'
vm_logfile='${vm_logfile}'
vm_vnc_port='${vm_vnc_port}'

bhyve_generate_acpi='${bhyve_generate_acpi}'
bhyve_wire_memory='${bhyve_wire_memory}'
bhyve_rts_keeps_utc='${bhyve_rts_keeps_utc}'
bhyve_force_msi_irq='${bhyve_force_msi_irq}'
bhyve_x2apic_mode='${bhyve_x2apic_mode}'
bhyve_mptable_gen='${bhyve_mptable_gen}'
bhyve_ignore_msr_acc='${bhyve_ignore_msr_acc}'

cd_vnc_wait="${cd_vnc_wait}"
bhyve_vnc_resolution="${bhyve_vnc_resolution}"
bhyve_vnc_tcp_bind="${bhyve_vnc_tcp_bind}"

vnc_password="${vnc_password}"
virtio_9p_args="${virtio_9p_args}"
bhyve_vnc_vgaconf="${bhyve_vnc_vgaconf}"

media_auto_eject="${media_auto_eject}"

nice='${nice}'
EOF

	checkpoint_args=

	if [ -n "${checkpoint}" ]; then
		CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
		CHECKPOINT="${CHECKPOINT_DIR}/${checkpoint}.ckp"
		if [ -r ${CHECKPOINT} ]; then
			${ECHO} "${MAGENTA}Checkpoint found, starting from: ${GREEN}${CHECKPOINT}${NORMAL}"
			checkpoint_args="-r ${CHECKPOINT}"
		else
			err 1 "${MAGENTA}Checkpoint not found: ${GREEN}${CHECKPOINT}${NORMAL}"
		fi
	fi

	${tmuxcmd} -2 -u new -d -s "cbsd-${jname}" "/bin/sh ${sharedir}/bhyverun.sh -c ${jailsysdir}/${jname}/bhyve.conf ${checkpoint_args}"

	if [ -n "${console_nmdm}" ]; then
		${tmuxcmd} select-window -t cbsd-${jname}

		for i in ${console_nmdm}; do
			${tmuxcmd} new-window -t ${con} "cu -l ${i} -s 9600"
			con=$(( con + 1 ))
		done

		con=0
		for i in ${console_nmdm}; do
			if [ $con -eq 0 ]; then
				${tmuxcmd} split-window -v -p 96 -t 0 "cu -l ${i} -s 9600" # DEBUG string
			else
				${tmuxcmd} new-window -t 1 "cu -l ${i} -s 9600"
			fi
			con=$(( con + 1 ))
		done

		${tmuxcmd} select-window -t cbsd-${jname}:0
	fi

	external_exec_master_script "master_poststart.d"

	# CBSD QUEUE
	if [ -x "${moduledir}/cbsd_queue.d/cbsd_queue" ]; then
		[ "${cbsd_queue_name}" != "none" ] && cbsd_queue cbsd_queue_name=${cbsd_queue_name} id=${jname} cmd=bstart status=2 data_status=1
	fi

	search_cmd="bhyve: ${jname}"
	strlen_search_cmd=$( strlen "${search_cmd}" )

	printf "${MAGENTA}Waiting for PID"
	for i in $( /usr/bin/seq 10 ); do
		vm_pid=$( /bin/ps axopid,ucomm,command -ww | while read pid ucomm command; do
			[ "${ucomm}" != "bhyve" ] && continue
			cmd_pref=$( substr --pos=0 --len=${strlen_search_cmd} --str="${command}" )

			if [ "${cmd_pref}" = "${search_cmd}" ]; then
				echo ${pid}
				break
			fi

			if echo "${command}" | /usr/bin/egrep -q -e " ${jname}"$ 2>/dev/null; then
				echo ${pid}
				break
			fi

		done )
		[ -n "${vm_pid}" ] && break
		sleep 1
		printf "."
	done

	[ -z "${vm_pid}" ] && vm_pid="0"

	echo

	${ECHO} "${MAGENTA}PID: ${GREEN}${vm_pid}${NORMAL}"
	cbsdsql local "UPDATE jails SET jid=\"${vm_pid}\" WHERE jname=\"${jname}\""

	# update state_time
	cbsdsql local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname=\"${jname}\"

	if [ "${vm_efi}" = "none" ]; then
		err 0 "${MAGENTA}Use ${GREEN}cbsd blogin ${jname}${MAGENTA} for attach to console${NORMAL}"
	else
		err 0 ""
	fi
}


# MAIN for multiple jails
TRAP=""
[ -z "${cbsd_queue_name}" ] && cbsd_queue_name="/clonos/bhyvevms/"

emulator="bhyve"	# for jname_is_multiple
jname_is_multiple

if [ $# -gt 1 -a -z "${jname}" -o -n "${jail_list}" ]; then
	# multiple astart always non interactive
	export inter=0
	# recursive
	if [ -n "${jail_list}" ]; then
		JLIST="${jail_list}"
	else
		JLIST=$*
	fi

	for jname in ${JLIST}; do
		[ "${jname}" = "inter=0" ] && continue
		TRAP="${TRAP} /bin/rm -f ${ftmpdir}/bstart.${jname}.$$;"
		trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
		if [ -n "${cbsd_queue_name}" ]; then
			/usr/sbin/daemon -p ${ftmpdir}/bstart.${jname}.$$ /usr/local/bin/cbsd bstart inter=${inter} jname=${jname} cbsd_queue_name="${cbsd_queue_name}"
		else
			/usr/sbin/daemon -p ${ftmpdir}/bstart.${jname}.$$ /usr/local/bin/cbsd bstart inter=${inter} jname=${jname}
		fi
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/bstart.${jname}.$$" ] && cbsd_pwait --pid=$( /bin/cat ${ftmpdir}/bstart.${jname}.$$ ) --timeout=${parallel}
		trap "" HUP INT ABRT BUS TERM EXIT
		# Artificial delay to create a sequence (for order compliance)
		# todo: determine VM complete starting
		sleep 12
	done

	wait_for_fpid -a start -t ${parallel}

	err 0 "${MAGENTA}Multiple bstart: ${GREEN}done${NORMAL}"
fi


# MAIN
. ${sharedir}/bhyve.conf		# only for for MYCOL variables: used in exports below
st_time=$( /bin/date +%s )
[ -z "${jname}" ] && jname=$1

. ${jrcconf}
if [ $? -eq 1 ]; then
	# remote start
	[ ${sqlreplica} -eq 0 ] && err 1 "${MAGENTA}No such domain: ${GREEN}${jname}${NORMAL}"
	remotenode=$( bwhereis ${jname} )
	[ -z "${remotenode}" ] && err 1 "${MAGENTA}No such domain: ${GREEN}${jname}${NORMAL}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			${ECHO} "${MAGENTA}Remote bstart: found on nodename ${GREEN}${nodename}${MAGENTA}. Skipped${NORMAL}"
			continue
		fi
		${ECHO} "${MAGENTA}Remote bstart: ${GREEN}${jname} ${MAGENTA}on${GREEN} ${i}${NORMAL}"
		rexe node=${i} cbsd bstart jname=${jname}
		if [ $? -eq 0 ]; then
			# updating state and put task for retrinv inventory
			${ECHO} "${MAGENTA}Updating inventory...${NORMAL}"
			task mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
		fi
	done
	exit 0
fi

default_profile="bhyve-default-default.conf"
readconf ${default_profile}

init_bhyve
init_systap

readconf vnc.conf
readconf bstart.conf

readconf ${default_profile}

. ${jrcconf}
[ $? -eq 1 ] && err 1 "${MAGENTA}No such jail: ${GREEN}${jname}${NORMAL}"

[ ${status} -eq 2 ] && err 1 "${MAGENTA}Jail in slave mode. Please ${GREEN}cbsd jswmode mode=master${MAGENTA} first${NORMAL}"

[ $jid -ne 0 ] && err 1 "${MAGENTA}Jail ${jname} already running, jid: ${GREEN}${jid}${NORMAL}"
[ "${emulator}" != "bhyve" ] && err 1 "${MAGENTA}Not bhyve mode${NORMAL}"
[ -z "${vm_ram}" -o -z "${vm_cpus}" -o -z "${vm_os_type}" ] && err 1 "${MAGENTA}Parameter is mandatory: ${GREEN}vm_ram, vm_cpus, vm_os_type${NORMAL}"
[ -z "${iso_auto_fetch}" ] && iso_auto_fetch=0
[ -z "${debug}" ] && debug=0

if [ ${vm_cpus} -gt ${ncpu} -a ${vm_cpus} -lt 16 ]; then
	${ECHO} "${MAGENTA}Warning! Current node cpu: ${GREEN}${ncpu}${MAGENTA}, guest cpu: ${GREEN}${vm_cpus}. ${MAGENTA}Overcommitting vCPUs can hurt perfomance.${NORMAL}"
elif [ ${vm_cpus} -lt 1 -o ${vm_cpus} -gt 16 ]; then
	err 1 "${MAGENTA}Valid number of guest CPUs within 1 - 16 range. Current vm_cpus: ${GREEN}${vm_cpus}${NORMAL}"
fi

# hardcoded first disk path from SQL. Todo: mark bootable disk(s)
MDFILE=$( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM bhyvedsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" LIMIT 1 2>/dev/null )

[ -z "${MDFILE}" ] && ${ECHO} "${MAGENTA}Warning: no any storage device found for this VM${NORMAL}"

if [ ! -f "${data}/${MDFILE}" -a ! -h "${data}/${MDFILE}" ]; then
	${ECHO} "${MAGENTA}No such ${data}/${MDFILE} but mdsize flags is not null.${NORMAL}"

	# if zfsfeat=1, try scan for zvol
	[ "${zfsfeat}" != "1" ] && break

	readconf zfs.conf
	. $zfstool
	DATA=$( /sbin/zfs get -Ho value name ${jaildatadir} 2>/dev/null )

	[ -z "${DATA}" ] && break

	for lunname in $( /usr/bin/seq 0 10 ); do
		if [ -r /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ]; then
			/bin/ln -sf /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ${data}/dsk${lunname}.vhd
			${ECHO} "${MAGENTA}Found zvol and create symlink: ${data}/dsk${lunname}.vhd -> ${DATA}/bcbsd-${jname}-dsk${lunname}.vhd"
		fi
	done
fi


# export variables for external hooks
export jname=${jname}

for _i in ${JARG} ${MYCOL}; do
	T=
	eval T="\$$_i"
	export ${_i}="${T}"
done

# test for incorrect state
if [ ${status} -eq 3 ]; then
	cbsdsql local UPDATE jails SET maintenance=\"${comment}\" WHERE jname=\"${jname}\"
	comment="cbsdsql local SELECT maintenance FROM jails WHERE jname=\"${jname}\""
	if [ "${comment}" = "Stopping_VM" ]; then
		jswmode jname=${jname} mode=master comment='0'
	else
		${ECHO} "${MAGENTA}Bhyve in maintenance: ${GREEN}${comment}${NORMAL}"
		err 1 "${MAGENTA}Please finish maintenance and switch mode via: ${GREEN}jswmode jname=${jname} mode=master comment='0'${NORMAL}"
	fi
fi

start_bhyve
end_time=$( /bin/date +%s )
cbsdlogger NOTICE ${CBSD_APP}: bhyve domain ${jname} started in $(( end_time - st_time ))s

exit 0
