#!/usr/local/bin/cbsd
#v11.1.0
MYARG=""
MYOPTARG="jname inter"
MYDESC="Start jail"
ADDHELP="inter=0 to prevent any questions and to accept answers by default\n"
CBSDMODULE="jail"
EXTHELP="wf_jstop_jstart.html"

. ${subr}
. ${system}
. ${strings}
. ${tools}
. ${workdir}/universe.subr
readconf buildworld.conf
readconf jail-freebsd-default.conf

[ -z "${1}" ] && select_jail_by_list -s "List of offline jail" -a "Off" -r ${sqlreplica}
init $*

# for external_exec-related command
. ${workdir}/jcreate.subr

# check environment scripts from tests dir
# when error code != 0 - exit
# $1 - is source dir name ( tests )
check_environment_script()
{
	local _res
	local _dir="${1}"
	local _testfiles

	[ -z "${_dir}" ] && return 1

	local _srcdir="${jailsysdir}/${jname}/${_dir}"

	[ ! -d "${_srcdir}" ] && return 0
	[ -z "$( /bin/ls ${_srcdir}/ )" ] && return 0

	_testfiles=$( /usr/bin/find "${_srcdir}" -type l -or -type f -depth 1 -maxdepth 1 -exec /usr/bin/basename {} \; )

	for _file in ${_testfiles}; do
		${ECHO} "${MAGENTA}Check environment script: ${GREEN}${_file}${NORMAL}"
		env jname=${jname} ${_srcdir}/${_file}
		[ $? -ne 0 ] && err 1 "${MAGENTA}Error: ${GREEN}${_file}${NORMAL}"
	done
}

exec_cbsdjail_first_boot()
{
	if [ -f ${path}/etc/rc.cbsdjail_first_boot ]; then
		local CBSDPATH="${PATH}"
		# reset CBSD PATH
		export PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
		/usr/local/bin/cbsd jexec jname=${jname} /bin/sh /etc/rc.cbsdjail_first_boot
		# restore CBSD PATH
		export PATH="${CBSDPATH}"
		/bin/rm -f ${path}/etc/rc.cbsdjail_first_boot
	fi
}

# return in $FWNUM first free ipfw num
# return 0 if not available
get_first_available_fwcount()
{
	for i in $( /usr/bin/seq ${fwcount_st} ${fwcount_end} ); do
		/sbin/ipfw -q show ${i} > /dev/null 2>&1
		if [ $? -ne 0 ]; then
			FWNUM=$i
			return 0
		fi
	done

	[ $i -eq ${fwcount_end} ] && FWNUM=0
}

fwcounter()
{
	local _in _out

	[ ${myjid} -eq 0 ] && return 0
	if [ "${ipfw_enable}" = "0" -a -z "`sysctl -n net.inet.ip.fw.enable 2>/dev/null`" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: skip for fwcounter setup: ipfw is not enabled
		return 0
	fi

	if [ ${freebsdhostversion} -gt 1100120 ]; then
		COMMENT="// Setup by CBSD jstart: ${jname}"
	else
		COMMENT=
	fi

	${ECHO} "${MAGENTA}Setup: ${GREEN}FW counter${NORMAL}"

	# FWIN
	get_first_available_fwcount
	if [ ${FWNUM} -eq 0 ]; then
		cbsdlogger WARNING ${CBSD_APP}: jail ${jname} warning: FW counter is not available in ${fwcount_st}-${fwcount_end} range. skip for fwcounter setup
		${ECHO} "${MAGENTA}Warning: FW counter not available in ${fwcount_st}-${fwcount_end} range. Skip${NORMAL}"
		_in=
	else
		_in=${FWNUM}
		/sbin/ipfw -q add ${FWNUM} count ip from any to me jail ${myjid} ${COMMENT} && echo ${FWNUM} > ${ftmpdir}/${jname}-fwin
	fi

	#FWOUT
	get_first_available_fwcount
	if [ ${FWNUM} -eq 0 ]; then
		cbsdlogger WARNING ${CBSD_APP}: jail ${jname} warning: FW counter is not available in ${fwcount_st}-${fwcount_end} range. skip for fwcounter setup
		${ECHO} "${MAGENTA}Warning: FW counter not available in ${fwcount_st}-${fwcount_end} range. Skip${NORMAL}"
		_out=
	else
		_out=${FWNUM}
		/sbin/ipfw -q add ${FWNUM} count ip from me to any jail ${myjid} ${COMMENT} && echo ${FWNUM} > ${ftmpdir}/${jname}-fwout
	fi

	[ -n "${_in}" -o -n "${_out}" ] && cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: setup IPFW rule counter for in/out with follow ipfw rule number: ${_in}/${_out}
}

# prepare symlink for emulator and
# check that the emulator is able to execute commands
emulator_exec_check()
{
	local _res _ret

	[ -z "${emulator}" ] && return 0
	[ "${emulator}" = "bhyve" ] && return 0
	[ "${emulator}" = "jail" ] && return 0
	[ "${emulator}" = "0" ] && return 0

	preparebase dst=${BASE_DIR} emulator=${emulator}

	_res=$( /usr/sbin/chroot ${path} /usr/local/bin/${emulator} /bin/sh -c "echo ping" 2>&1 )
	_ret=$?
	[ ${_ret} -ne 0 ] && printf "${_res}"
	return ${_ret}
}


start_bhyve()
{
	${ECHO} "${MAGENTA}For bhyve mode use: ${GREEN}cbsd bstart jname=${jname} ${MAGENTA}instead.${NORMAL}"
	exit 0
}

# chase 'grep -A28 ^20140930: /usr/ports/CHANGES'
set_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	[ ${applytpl} -eq 0 ] && return 0
	[ "${ver}" = "empty" ] && return 0

	local nodever=$( ${miscdir}/elf_tables --freebsdver /bin/sh 2>/dev/null )
	local jailver=$( ${miscdir}/elf_tables --freebsdver ${path}/bin/sh 2>/dev/null )
	local osversion=$( ${miscdir}/elf_tables --ver ${path}/bin/sh 2>/dev/null )

	[ "${nodever}" = "${jailver}" ] && return 0

	${ECHO} "${MAGENTA}Applying /usr/ports/CHANGES -A28 ^20140930 work-around${NORMAL}"
	# we need put fake UNAME_r-stuff environment for correct works of port system
	if ! is_number ${ver}; then
		# is stable
		local release="${ver}.0-STABLE"
	else
		# is release
		local release="${ver}-RELEASE"
	fi

	# make backup and add records
	[ -f "${data}/etc/make.conf" ] && /bin/cp ${data}/etc/make.conf ${data}/etc/make.conf.bak

	/bin/cat >> ${data}/etc/make.conf <<EOF

OSVERSION+= ${osversion}			# ${_tpl}
UNAME_ENV+= OSVERSION=${osversion}		# ${_tpl}
UNAME_ENV+= UNAME_s=FreeBSD			# ${_tpl}
UNAME_ENV+= UNAME_r=${release}			# ${_tpl}
UNAME_ENV+= UNAME_v="\${UNAME_s} \${UNAME_r}"	# ${_tpl}
.MAKEFLAGS: \${UNAME_ENV}			# ${_tpl}
MAKE_ENV+= \${UNAME_ENV}			# ${_tpl}
CONFIGURE_ENV+= \${UNAME_ENV}			# ${_tpl}
SCRIPTS_ENV+= \${UNAME_ENV}			# ${_tpl}

EOF

	/bin/cat >> ${data}/etc/profile << EOF

OSVERSION=${osversion}; export OSVERSION		# ${_tpl}
UNAME_s=FreeBSD; export UNAME_s				# ${_tpl}
UNAME_r=${release}; export UNAME_v			# ${_tpl}
UNAME_v="\${UNAME_s} \${UNAME_r}"; export UNAME_v	# ${_tpl}

EOF

	/bin/cat >> ${data}/etc/csh.cshrc << EOF

setenv OSVERSION ${osversion}			# ${_tpl}
setenv UNAME_s FreeBSD				# ${_tpl}
setenv UNAME_r ${release}			# ${_tpl}
setenv UNAME_v "\${UNAME_s} \${UNAME_r}"	# ${_tpl}

EOF
}

unset_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	if ${GREP_CMD} "${_tpl}" ${data}/etc/make.conf >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/make.conf ${data}/etc/make.conf.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/make.conf.bak | ${GREP_CMD} "." > ${data}/etc/make.conf
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/profile >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/profile ${data}/etc/profile.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/profile.bak | ${GREP_CMD} "." > ${data}/etc/profile
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/csh.cshrc >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/csh.cshrc ${data}/etc/csh.cshrc.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/csh.cshrc.bak | ${GREP_CMD} "." > ${data}/etc/csh.cshrc
	fi
}


emulator="jail"	# for jname_is_multiple
jname_is_multiple

# MAIN for multiple jails
if [ $# -gt 1 -a -z "${jname}" -o -n "${jail_list}" ]; then
	# multiple astart always non interactive
	export inter=0
	# recursive
	if [ -n "${jail_list}" ]; then
		JLIST="${jail_list}"
	else
		JLIST=$*
	fi

	for jname in ${JLIST}; do
		[ "${jname}" = "inter=0" ] && continue
		/usr/sbin/daemon -p ${ftmpdir}/jstart.${jname}.$$ /usr/local/bin/cbsd jstart inter=0 jname=${jname}
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( /bin/cat ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}
	done

	wait_for_fpid -a start -t ${parallel}

	err 0 "${MAGENTA}Multiple jstart: ${GREEN}done${NORMAL}"
fi


# MAIN
st_time=$( /bin/date +%s )
[ -z "$jname" ] && jname=$1
. ${jrcconf}

if [ $? -eq 1 ]; then
	# remote start
	[ ${sqlreplica} -eq 0 ] && err 1 "${MAGENTA}No such jail: ${GREEN}${jname}${NORMAL}"
	remotenode=$( jwhereis ${jname} )
	[ -z "${remotenode}" ] && err 1 "${MAGENTA}No such jail: ${GREEN}${jname}${NORMAL}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			${ECHO} "${MAGENTA}Remote jstart: found on nodename ${GREEN}${nodename}${MAGENTA}. Skipped${NORMAL}"
			continue
		fi
		${ECHO} "${MAGENTA}Remote jstart: ${GREEN}${jname} ${MAGENTA}on${GREEN} ${i}${NORMAL}"
		rexe node=${i} cbsd jstart jname=${jname}
		if [ $? -eq 0 ]; then
			# updating state and put task for retrinv inventory
			${ECHO} "${MAGENTA}Updating inventory...${NORMAL}"
			task mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
		fi
	done
	exit 0
fi

over="${ver}"

# Determine stable value. Must be after buildconf
strpos --str="${over}" --search="."

# auto-detect for stable/release
pos=$?
if [ ${pos} -eq 0 ]; then
	stable=1
	ostable=1
else
	stable=0
	ostable=0
fi

[ ${status} -eq 2 ] && err 1 "${MAGENTA}Jail in slave mode. Please ${GREEN}cbsd jswmode mode=master${MAGENTA} first${NORMAL}"
[ ${status} -eq 3 ] && err 1 "${MAGENTA}Jail in maintenance mode${NORMAL}"
[ ${jid} -ne 0 ] && err 1 "${MAGENTA}Jail ${jname} already running, jid: ${GREEN}${jid}${NORMAL}"

if [ "${vnet}" = "1" -a "${vimage_feature}" = "0" ]; then
	${ECHO} "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} have vnet=1 flags but your kernel is not support VIMAGE${NORMAL}"
	${ECHO} "${MAGENTA}Please recompile kernel with: ${GREEN}options VIMAGE${NORMAL}"
	sleep 3
	vnet=0
fi

[ "${emulator}" = "bhyve" -a -z "${mdsize}" ] && err 1 "${MAGENTA}Bhyve required for file image${NORMAL}"
[ -z "${cbsd_queue_name}" ] && cbsd_queue_name="/clonos/jailscontainers/"

TRAP=""

#Check for shared lock
jaillock="${jailsysdir}/${jname}/locked"
if [ -f "${jaillock}" ]; then
	masterhost=$( /bin/cat ${jaillock} )
	if [ "${masterhost}" = "${nodename}" -o -z "${masterhost}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: remove my stale lock file: ${jaillock}
		${ECHO} "${MAGENTA}Remove my stale lock file: ${GREEN}${jaillock}${NORMAL}"
		/bin/rm -f ${jaillock}
	else
		cur_time=$( /bin/date +%s )
		eval $( /usr/bin/stat -s ${jaillock} )
		difftime=$(( ( cur_time - st_mtime ) / 60 ))

		# 30 minutes outdated lock
		if [ ${difftime} -gt 30 ]; then
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file but lock file age is too old: ${difftime}. Removing!
			${ECHO} "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} locked by ${GREEN}${masterhost}${MAGENTA} node${NORMAL}"
			${ECHO} "${MAGENTA}But lock age is too old: ${difftime} min. Removing!!!${NORMAL}"
			/bin/rm -f ${jaillock}
		else
			# still fresh
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file and lock file age time is fresh: ${difftime}.
			${ECHO} "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} locked by ${GREEN}${masterhost}${MAGENTA} node, lock age: ${difftime} min.${NORMAL}"
			log_err 1 "${MAGENTA}You may remove the lockfile if you believe that jail is not running on this node: ${GREEN}rm -f ${jaillock}${NORMAL}"
		fi
	fi
fi

TRAP="${TRAP} /bin/rm -f ${jaillock};"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

#Check for md vnode backend
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	MDFILE="${jailsysdir}/${jname}/image.dat"
	if [ ! -f "${MDFILE}" -a ! -h "${MDFILE}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail: ${jname}: no such ${MDFILE} but mdsize flags is not null, skip
		${ECHO} "${MAGENTA}No such ${MDFILE} but mdsize flags is not null. Skip${NORMAL}" && continue
	fi
fi

# CBSD QUEUE
if [ -x "${moduledir}/cbsd_queue.d/cbsd_queue" ]; then
	[ "${cbsd_queue_name}" != "none" ] && cbsd_queue cbsd_queue_name=${cbsd_queue_name} id=${jname} cmd=jstart status=1
fi

[ "${emulator}" = "bhyve" ] && start_bhyve

init_target_arch
init_basedir

if [ ${baserw} -eq 0 ]; then
	if [ -r "${BASE_DIR_LOCKFILE}" ]; then
		locked_by=$( /bin/cat ${BASE_DIR_LOCKFILE} )
		log_err 1 "${MAGENTA}basedir locked: ${GREEN}${BASE_DIR}${MAGENTA}, by pid: ${GREEN}${locked_by}${MAGENTA}. Please try later${NORMAL}"
	fi
fi

jcleanup jname=${jname}
[ ! -d "${path}" ] && /bin/mkdir ${path}

check_environment_script "tests"

[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && /usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-jailnic.schema jailnic

case ${ip4_addr} in
	[Dd][Hh][Cc][PP])
		ip4_addr=$( dhcpd )
		[ $? -eq 2 ] && log_err 1 "${MAGENTA}No free IP address for DHCP in nodeippool${NORMAL}"
		cbsdlogger NOTICE ${CBSD_APP}: set ip4_addr for ${jname} via DHCP: ${ip4_addr}
		jset jname=${jname} ip4_addr="${ip4_addr}"
		;;
esac

geniplist ${ip4_addr}

if [ -n "${interface}" -a "${interface}" != "0" -a "${vnet}" -eq "0" ]; then
	### CHECK FOR IP ####
	for ips in ${IPS}; do
		iptype "${ips}" ||true
		[ -z "${IWM}" ] && continue
		[ -n "${VHID}" ] && continue
		#prevent to use nodeip
		[ "${IWM}" = "${nodeip}" ] && err 1 "${MAGENTA}Error: Jail can not take nodeip when interface is not equal 0: ${GREEN}${nodeip}${NORMAL}"
		checkip ip=${IWM} check=1
		IPUSE=$?
		case ${IPUSE} in
			0)
				${ECHO} "${MAGENTA}ip ${IWM} not in pool range${NORMAL}"
				continue
			;;
			2)
				${ECHO} "${MAGENTA}Ip ${IWM} already exists in LAN${NORMAL}"
				continue
			;;
		esac
	done
fi

# args for makejconf
epairb_list=

# extract and export/rewrite interface variable from jailnic for vnet-based jails
if [ "${vnet}" = "1" ]; then
	# for vnet we can make another action
	. ${vimageconf}

	interfaces=$( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT name FROM jailnic | while read _int; do
		printf "${_int} "
	done ) || err 1 "${MAGENTA}jstart: error get interfaces name for vnet nic map${NORMAL}"

	eth_seq=0
	printf "${MAGENTA}Create epair: "

	for i in ${interfaces}; do

		nic_hwaddr=
		nic_parent=

		nic_parent=$( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT nic_parent FROM jailnic WHERE name=\"${i}\" )

		# when ip=0 and interface=auto we must use default interface for upstream
		[ "${nic_parent}" = "0" -o "${nic_parent}" = "auto" ] && nic_parent=$( getnics-by-ip ip=0.0.0.0 )
		if ! mybridge=$( get_my_device bridge ${nic_parent} ); then
			err 1 "${MAGENTA}Error: Cant get_my_device for ${nic_parent}: ${mybridge}${NORMAL}"
		fi
		myepair=$( get_my_epair ${mybridge} )
		[ $? -eq 1 ] && err 1 "${MAGENTA}Error: Cant get_my_epair by: ${GREEN}${mybridge}${NORMAL}"
		if [ -z "${epairb_list}" ]; then
			epairb_list="${myepair}b"
		else
			epairb_list="${epairb_list},${myepair}b"
		fi
		printf "${LYELLOW}${myepair}:${GREEN}${nic_parent} "
		/sbin/ifconfig ${myepair}a description ${jname}-eth${eth_seq} up
		eth_seq=$(( eth_seq + 1 ))
		TRAP="${TRAP} /sbin/ifconfig ${myepair}a destroy;"
		trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

		nic_hwaddr=$( cbsdsql ${jailsysdir}/${jname}/local.sqlite SELECT nic_hwaddr FROM jailnic WHERE name=\"${i}\" )

		# MAC MGMT
		if [ "${nic_hwaddr}" = "0" ]; then
			nic_hwaddr=$( mac_gen 00:a0:98 )
			cbsdsql ${jailsysdir}/${jname}/local.sqlite "UPDATE jailnic SET nic_hwaddr=\"${nic_hwaddr}\" WHERE name=\"${i}\""
		fi

		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: set hwaddr for vnet interface ${myepair}b: ${nic_hwaddr}
		/sbin/ifconfig ${myepair}b ether ${nic_hwaddr}
	done

	${ECHO} "${NORMAL}"

fi	# vnet -eq 1

#test for zfs mounted & mount if not
case ${zfsfeat} in
	1)
		. ${zfstool}
		zfsmnt ${data}
		[ $? -eq 2 ] && /sbin/zfs mount "${ZPOOL}"
	;;
esac

if [ "${ver}" != "empty" ]; then
	if [ ${baserw} -eq 1 ]; then
		path=${data}
		[ ! -f "${path}/bin/sh" ] && switch_baserw ${path} 1
		[ -f ${mount_fstab} ] && /bin/rm -f ${mount_fstab}
	fi

	if [ ${baserw} -eq 0 -a ! -f "${mount_fstab}" ]; then
		switch_baserw ${path} 2
	fi
fi

# MD area
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	cbsd mountmd mdfile="${MDFILE}" jroot="${data}"
fi

if [ "${ver}" != "empty" ]; then
	[  -d "${data}/etc" -o -d "${data}/bin" ] || err 1 "${MAGENTA}No such data structure: ${GREEN}${data}${NORMAL}"
fi

# cp local default resolv.conf skel
if [ ${floatresolv} -eq 1 -a "${ver}" != "empty" ]; then
	makeresolv jname=${jname}
fi

if [ ${applytpl} -eq 1 -a "${ver}" != "empty" ]; then
	${SYSRC_CMD} -qf ${data}/etc/rc.conf hostname="${host_hostname}" >/dev/null
fi

MOUNTOPT=""

if [ "${mount_src}" = "1" ]; then
	SRCDIR="${srcdir}/src_${ver}/src"
	MOUNTOPT="${MOUNTOPT} -s ${SRCDIR}"
fi

if [ "${mount_obj}" = "1" ]; then
	SRCDIR="${srcdir}/obj_${arch}_${ver}/obj"
	MOUNTOPT="${MOUNTOPT} -o ${SRCDIR}"
fi

if [ "${mount_kernel}" = "1" ]; then
	if [ -d ${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel ]; then
		SRCDIR="${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel"
		MOUNTOPT="${MOUNTOPT} -k ${SRCDIR}"
	fi
fi

if [ "${mount_ports}" = "1" ]; then
	MOUNTOPT="${MOUNTOPT} -p /usr/ports -d ${data}/var/cache/distfiles"
fi

[ "${ver}" != "empty" ] && mountbase -v ${ver} -a ${arch} ${MOUNTOPT}

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
with_zfs_attach="${jailsysdir}/${jname}/with_zfs_attach"
[ -r ${with_zfs_attach} ] && /bin/rm -f ${with_zfs_attach}

mount_jail_fstab
export_jail_data_for_external_hook
external_exec_master_script "master_prestart.d"

if [ "${ver}" = "empty" ]; then
	path="/"
	exec_start="${jailsysdir}/${jname}/run.sh"
else
	[ ! -d "${data}/var/cache/pkg" ] && /bin/mkdir -p "${data}/var/cache/pkg"
fi

#determine that jail is FreeBSD. Useful for vnet operation in makejconf and
is_freebsd=0

if [ ${baserw} -eq 1 ]; then
	elftest=${data}/bin/sh
else
	elftest="${BASE_DIR}/bin/sh"
fi
[ -f "${elftest}" ] && osname=$( ${miscdir}/elf_tables --osname ${elftest} )

[ "${osname}" = "freebsd" ] && is_freebsd=1

# check for freshed version of the base via elf from /bin/sh
if [ ${is_freebsd} -eq 1 -a ${baserw} -eq 1 ]; then
	if [ -n ${BASE_DIR} -a -f "${BASE_DIR}/bin/sh" ]; then
		baseelf=$( ${miscdir}/elf_tables --ver ${BASE_DIR}/bin/sh 2>/dev/null )
		jailelf=$( ${miscdir}/elf_tables --ver ${elftest} 2>/dev/null )
		if [ ${baseelf} -gt ${jailelf} ]; then
			${ECHO} "${BOLD}Notice: ${MAGENTA}You have a more recent version of the base in ${BASE_DIR} (${GREEN}${baseelf}${MAGENTA}/${GREEN}${jailelf}${MAGENTA}).${NORMAL}"
			${ECHO} "${BOLD}Notice: ${MAGENTA}Please consider upgrading jail base via ${GREEN}cbsd jupgrade${NORMAL}"
		fi
	fi
fi

set -e
makejconf jname=${jname} out=${ftmpdir}/${jname}.conf ip6wa=${HAVE_IPV6} epair=${epairb_list} fbsd=${is_freebsd}
set +e

#rctl/limits area
jrctl jname=$jname mode=set
#. ${workdir}/rctl.subr
[ -r "${jailsysdir}/${jname}/helpers/jrctl.sqlite" ] && nice=$( cbsdsql ${jailsysdir}/${jname}/helpers/jrctl.sqlite "SELECT cur FROM forms WHERE param=\"nice\"" )
[ -z "${nice}" ] && nice="0"
[ "${nice}" != "0" ] && ${ECHO} "${MAGENTA}jail renice: ${GREEN}${nice}${NORMAL}"

unset_changes_20140930
set_changes_20140930

${ECHO} "${MAGENTA}Starting jail: ${GREEN}${jname}, parallel timeout=${parallel}${NORMAL}"
TRAP="${TRAP} /bin/rm -f ${ftmpdir}/jstart.${jname}.$$;"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

# check that the emulator is able to execute commands
res=$( emulator_exec_check )

ret=$?

if [ ${ret} -ne 0 ]; then
	$ECHO "${MAGENTA}Emulator error: ${GREEN}${res}${NORMAL}"
	jcleanup jname="${jname}"
	exit 0
fi

/usr/sbin/daemon -p ${ftmpdir}/jstart.${jname}.$$ /usr/bin/nice -n ${nice} /usr/sbin/jail -f ${ftmpdir}/${jname}.conf -c ${jname}
trap "" HUP INT ABRT BUS TERM EXIT

[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( /bin/cat ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}

### late start ###
# late exec_start: for zfsattached and vnet-based jails
# this is necessary for any operations that must be performed after 
# creating the jail but before running real exec_start
late_start=

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
if [ -r ${with_zfs_attach} -a ${is_freebsd} -eq 1 ]; then
	[ -r ${mount_fstab} ] && attachzfs fstab=${mount_fstab} jname=${jname}
	[ -r ${mount_fstab}.local ] && attachzfs fstab=${mount_fstab}.local jname=${jname}
	# real exec_start for zfs attached jail
	/usr/sbin/jexec ${jname} /sbin/zfs mount -a
	late_start="${exec_start}"
fi

# for VNET-based jail we also need special route
# due to jail.conf doesn't support for multiple NICs
if [ ${vnet} -eq 1 ]; then
	OIFS="${IFS}"
	IFS=","
	eth_seq=0
	for i in ${epairb_list}; do
		[ -z "${i}" ] && continue
		IFS="${OIFS}"
		/sbin/ifconfig ${i} up
		# we need setup lock here due to ethX name possible conflict with other jail
		while [ -r ${tmpdir}/eth${eth_seq}.locked ]; do
			${ECHO} "${MAGENTA}jstart: eth${eth_seq} locked, waiting..${NORMAL}"
			sleep 1
		done
		# make lock
		echo "${jname}" > ${tmpdir}/eth${eth_seq}.locked
		/sbin/ifconfig ${i} name eth${eth_seq} && /sbin/ifconfig eth${eth_seq} vnet ${jname}
		# release lock
		/bin/rm -f ${tmpdir}/eth${eth_seq}.locked
		eth_seq=$(( eth_seq + 1 ))
		IFS=","
	done
	IFS="${OIFS}"
	late_start="${exec_start}"
fi

if [ -n "${late_start}" ]; then
	/usr/sbin/jexec ${jname} ${late_start}
fi
### /late start ###

# update state_time
cbsdsql local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname=\"${jname}\"

# make id file
UNDHOSTNAME=$( echo ${hostname} | /usr/bin/tr  "." "_" )
FID="/var/run/jail_${UNDHOSTNAME}.id"
echo ${ST} > ${FID}
exec_cbsdjail_first_boot

external_exec_script "start.d"
external_exec_master_script "master_poststart.d"

get_jid

if [ ${myjid} -gt 0 ]; then
	status="1"
	# make shared lock
	echo "${nodename}" > ${jaillock}
	/usr/sbin/chown ${cbsduser}:${cbsduser} ${jaillock}
else
	status="0"
	# looks like jail start is failed, execute post script
	exec_master_poststop
	external_exec_master_script "master_poststop.d"
	exec_poststop
	jcleanup jname=${jname}
fi

cbsdsql local UPDATE jails SET jid=${myjid},status=${status} where jname=\"${jname}\"
/bin/rm -f ${ftmpdir}/${jname}.conf
fwcounter
expose jname=${jname} mode=apply
[ ${myjid} -gt 0 -a ${cpuset} != "0" ] && /usr/bin/cpuset -c -l ${cpuset} -j ${myjid}

# VNC auto start
if [ -x "${moduledir}/vncterm.d/cbsdvnc" ]; then
	${ECHO} "${MAGENTA}vncterm is installed, launch vncterm session...${NORMAL}"
	/usr/sbin/daemon -f /usr/local/bin/cbsd vncterm jname=${jname} mode=run addr=127.0.0.1 >/dev/null 2>&1
fi

# CBSD QUEUE
if [ -x "${moduledir}/cbsd_queue.d/cbsd_queue" ]; then
	[ "${cbsd_queue_name}" != "none" ] && cbsd_queue cbsd_queue_name=${cbsd_queue_name} id=${jname} cmd=jstart status=2 data_status=1
fi

end_time=$( /bin/date +%s )

cbsdlogger NOTICE ${CBSD_APP}: jail ${jname} started in $(( end_time - st_time ))s

exit 0
