#!/bin/bash
#
# mlx4_vnic       Bring up/down host-admin vnic
#
# chkconfig: - 06 94
# description: Activates/Deactivates mlx4 virtual network interfaces configured to \
#              start at boot time.
#
### BEGIN INIT INFO
# Provides: mlx4_vnicd
# Required-Start: openibd
# Should-Start:
# Required-Stop: $null
# Should-Stop:
# Default-Start:  2 3 5
# Default-Stop: 0 1 2 6
# Short-Description: Bring up/down vnic
# Description: Bring up/down vnic
### END INIT INFO

CWD=`pwd`
TMP=/tmp

VERBOSE=${VERBOSE:-"no"}
config=${config:-"/etc/infiniband/mlx4_vnic.conf"}
min_num_of_params=5
max_num_of_params=9
created=0
deleted=0
failed=0
invalid=0
add_timeout=5
del_timeout=10
conf_dir_RH='/etc/sysconfig/network-scripts/'
conf_dir_SLES='/etc/sysconfig/network/'
vnic_id_max=16384
na_str="N/A"
pkey_default="0xffff"
prog=/sbin/mlx4_vnicd
bprog=`basename $prog`

# Check that we are root ... so non-root users stop here
if test `id -u` != 0; then
	echo "You must be root"
	exit 1
fi

# create temporary file for results
conf_file=`mktemp $TMP/mlx4_vnic_conf.XXXXXXXXXXX`

# abort function to cleanup temp files
abort() {
	if [ -f $conf_file ]; then
		\rm -f $conf_file &> /dev/null
	fi

	exit $1
}

# create temporary config file from all relevant ifcfg-ethX files
create_conf_file() {

	[ "$VERBOSE" = no ] || echo $"concatenate  $config to $conf_file"
	test -r $config && cat $config >> "$conf_file"

	if [ -d "$conf_dir_SLES" ]; then
		conf_dir="$conf_dir_SLES"
	else
		if [ -d "$conf_dir_RH" ]; then
			conf_dir="$conf_dir_RH"
		else
			return 1
		fi
	fi

	for file_name in `ls $conf_dir/ifcf*`
	do
		BXADDR=''
		BXEPORT=''
		VNICVLAN=''
		VNICIBPORT=''
		DEVICE=''
		HWADDR=''
                pkey_entry=''
		mac_entry=''
		vlan_entry=''
	
		IPADDR=''
		SHAREDHWADDR=''
	
		if (`grep BXADDR "$file_name" > /dev/null`); then
			source $file_name
	
			# if DEVICE attribute is not defined in file extract it from the file name
			if [ -z "$DEVICE" ]; then
				DEVICE=`echo $file_name | sed -n "s/.*ifcfg-\(eth[0-9]*\)*/\1/p"`
				#echo $DEVICE
			fi
	
			# check we have all the information we require
			if [ -n "$BXADDR" ] && [ -n "$BXEPORT" ] && [ -n "$VNICIBPORT" ] && [ -n "$DEVICE" ]; then
				#echo "All parameters configured. Conf file line:" 
				if [ -z "$HWADDR" ]; then
					#echo "NO MAC configured"
					mac_entry="mac=00:00:00:00:00:00"
				else
					mac_entry="mac=$HWADDR"
				fi
	
				if [ -z "$VNICVLAN" ]; then
					vlan_entry="vid=-1"
				else
					vlan_entry="vid=$VNICVLAN"
				fi

				if [ -z "$GW_PKEY" ]; then
					pkey_entry="pkey=$pkey_default"
				else
					pkey_entry="pkey=$GW_PKEY"
				fi

				vnic_id=`echo $DEVICE | sed -n "s/eth\([0-9]*\)*/\1/p"`
	
				line="name=$DEVICE $mac_entry ib_port=$VNICIBPORT $vlan_entry vnic_id=$vnic_id bx=$BXADDR eport=$BXEPORT $pkey_entry"
	
				if ! [ -z "$SHAREDHWADDR" ]; then
					line="name=$DEVICE $mac_entry ib_port=$VNICIBPORT $vlan_entry vnic_id=$vnic_id bx=$BXADDR ip=$IPADDR emac=$SHAREDHWADDR eport=$BXEPORT $pkey_entry"
				fi
	
				echo "$line" >> "$conf_file"
			else
				echo "$file_name: Missing parameters skipping"
			fi
		fi
	done

	num=`test -r $conf_file && cat $conf_file | grep -vE "^$|^#" | wc -l`
	if [ "$num" == "0" ]; then
		ret=1
	else
		ret=0
	fi

	return $ret
}

if ! ( test -r $config && cat $config | grep -vE "^$|^#" | grep vnic_id -q &> /dev/null); then
	[ "$VERBOSE" = no ] || echo $"no vnic_id tokens, call create_conf_file"
	create_conf_file
	if [ $? -eq 0 ]; then
		config=$conf_file
		# echo "Trying to use ifcfg-ethX files (temporary config file: $config)"
	elif [ "$1" == "start" ]; then
		# config is needed only for 'start' and
		echo "Configuration file does not exist, skip"
		abort 0
	else
		config=$conf_file
		[ "$VERBOSE" = no ] || echo $"perform action ($1) without config file"
	fi
fi

if [ ! -d "/sys/module/mlx4_vnic" ]; then
	echo "ERROR: mlx4_vnic module is not loaded"
	abort 2
fi

# Get a sane screen width
[ -z "${COLUMNS:-}" ] && COLUMNS=80

[ -z "${CONSOLETYPE:-}" ] && [ -x /sbin/consoletype ] && CONSOLETYPE="`/sbin/consoletype`"

if [ -f /etc/sysconfig/i18n -a -z "${NOLOCALE:-}" ] ; then
	. /etc/sysconfig/i18n
	if [ "$CONSOLETYPE" != "pty" ]; then
		case "${LANG:-}" in
			ja_JP*|ko_KR*|zh_CN*|zh_TW*)
				export LC_MESSAGES=en_US
				;;
			*)
				export LANG
				;;
		esac
	else
		export LANG
	fi
fi

# Read in our configuration
if [ -z "${BOOTUP:-}" ]; then
	if [ -f /etc/sysconfig/init ]; then
		. /etc/sysconfig/init
	else
		# This all seem confusing? Look in /etc/sysconfig/init,
		# or in /usr/doc/initscripts-*/sysconfig.txt
		BOOTUP=color
		RES_COL=60
		MOVE_TO_COL="echo -en \\033[${RES_COL}G"
		SETCOLOR_SUCCESS="echo -en \\033[1;32m"
		SETCOLOR_FAILURE="echo -en \\033[1;31m"
		SETCOLOR_WARNING="echo -en \\033[1;33m"
		SETCOLOR_NORMAL="echo -en \\033[0;39m"
		LOGLEVEL=1
	fi
	if [ "$CONSOLETYPE" = "serial" ]; then
		BOOTUP=serial
		MOVE_TO_COL=
		SETCOLOR_SUCCESS=
		SETCOLOR_FAILURE=
		SETCOLOR_WARNING=
		SETCOLOR_NORMAL=
	fi
fi

if [ "${BOOTUP:-}" != "verbose" ]; then
	INITLOG_ARGS="-q"
else
	INITLOG_ARGS=
fi

echo_success() {
	echo -n $@
	[ "$BOOTUP" = "color" ] && $MOVE_TO_COL
	echo -n "[  "
	[ "$BOOTUP" = "color" ] && $SETCOLOR_SUCCESS
	echo -n $"OK"
	[ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL
	echo -n "  ]"
	echo -e "\r"
	return 0
}

echo_failure() {
	echo -n $@
	[ "$BOOTUP" = "color" ] && $MOVE_TO_COL
	echo -n "["
	[ "$BOOTUP" = "color" ] && $SETCOLOR_FAILURE
	echo -n $"FAILED" 
	[ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL
	echo -n "]"
	echo -e "\r"
	return 1
}

echo_warning() {
	echo -n $@
	[ "$BOOTUP" = "color" ] && $MOVE_TO_COL
	echo -n "["
	[ "$BOOTUP" = "color" ] && $SETCOLOR_WARNING
	echo -n $"WARNING"
	[ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL
	echo -n "]"
	echo -e "\r"
	return 1
}

get_random_vnic_id()
{
	i=0
	while true; do
		num=$RANDOM
		let "num%=$vnic_id_max"
		if ! ( grep vnic_id /sys/module/mlx4_vnic/vnic*-cmd* | grep -w vnic_id=$i ); then
			break
		fi
	done
	echo $num
}

get_random_eth_name()
{
	list=`/sbin/ip link | grep ^[0-9] | awk '{print $2}' | tr -d :`
	i=0
	while true; do
		name=eth$i
		if ! ( echo $list | grep -w $name -q ); then
			break
		fi
		i=$[$i+1]
	done
	echo $name
}

get_random_mac()
{
	a1=`echo $RANDOM | cut -b1-2`
	a2=`echo $RANDOM | cut -b1-2`
	a3=`echo $RANDOM | cut -b1-2`
	a4=`echo $RANDOM | cut -b1-2`
	a5=`echo $RANDOM | cut -b1-2`
	a6=`echo $RANDOM | cut -b1-2`

	let rc=a6%2
	if [ $rc ]; then
		let a6=$a6+1
	fi

	echo "$a1:$a2:$a3:$a4:$a5:$a6"
	return 0
}

is_mac_unicast()
{
	mcast_byte = `echo $1 | cut -b1-2`
	# if mcast_byte is odd, mac is not unicast
	let rc=mcast_byte%2
	return !$rc
}

is_mac()
{
	if [ -z `echo $1 | sed -n "/^\([0-9a-fA-F][0-9a-fA-F]:\)\{5\}[0-9a-fA-F][0-9a-fA-F]$/p"` ]; then
		return 1
	fi

	return 0
}

is_integer()
{
	printf "%s\n" $1 |grep -E "^[+-]?[0-9]+$" > /dev/null
	return $?
}

is_ip()
{
	local  ip=$1
	local  stat=1

	if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
		OIFS=$IFS
		IFS='.'
		ip=($ip)
		IFS=$OIFS
		[[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
			&& ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
		stat=$?
	fi

	return $stat
}

check_value() {
	line=$1
	parameter=$2
	value=$3
	shift 3

	if [ "$parameter" != "vid" ] && [ "$parameter" != "pkey" ]; then
		if [[ -z "$value" ]]; then
			return 1
		fi
	fi

	case $parameter in
		ib_port)
			if ! [[ $value =~ ^mlx4_[0-9a-fA-F]+:[1-2] ]]; then
				return 1
			fi
		;;
		name)
			# case $value in
			# 	eth* | auto)
			# 	;;
			# 	*)
			# 	return 1
			# 	;;
			# esac
		;;
		mac)
			if [ "$value" != auto ] && ( ! is_mac $value ) && ( ! is_mac_unicast $value ); then
				[ $VERBOSE = no ] || echo "ERROR: invalid mac=$mac"
				return 1
			fi
		;;
		vnic_id)
			if [ "$value" != auto ] && ( ! is_integer $value ) && ( ! "$value" -ge $vnic_id_max ); then
				[ $VERBOSE = no ] || echo "ERROR: invalid vnic_id=$vnic_id"
				return 1
			fi
		;;
		vid)
			if [ ! -z "$value" ] && [ "`echo $value | tr A-Z a-z`" != all ] && ( ! is_integer $value ); then
				return 1
			fi
		;;
		bx)
			if [ "$bx" == "00:00:00:00:00:00:00:00" ] || [ "$bx" == "unnamed_system" ]; then
				[ "$VERBOSE" = no ] || echo $"Invalid bx format: $bx"
				return 1
			fi
		;;
		ip)
			if ! is_ip $value; then
				return 1
			fi
		;;
		eport)
			# gw_port_name is no longer than 8 bytes
			if [ ${#value} -gt 8 ]; then
				return 1
			fi
		;;
		pkey)
			if [ ! -z "$value" ]; then
				if ! [[ $value =~ 0x[0-9a-fA-F]+ ]]; then
					return 1
				fi
			fi
		;;
		*)
		# Unsupported parameter
		return 1
		;;
	esac

	return 0
}

check_config() {

	# check prog lines
	num=`cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | grep ^$bprog= | wc -l`
	if [ $num -gt 1 ]; then
		[ "$VERBOSE" = no ] || echo $"too many lines for $bprog ($num)"
		echo_failure $"Checking configuration file:"
		abort 30
	fi

	# check that vnic_id is unique or auto
	num=` cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/vnic_id/ and not /auto/);' | wc -l`
	numu=`cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/vnic_id/ and not /auto/);' | sort -u | wc -l`
	if [ $num -ne $numu ]; then
		[ "$VERBOSE" = no ] || echo $"vnic_id should be unique"
		echo_failure $"Checking configuration file:"
		abort 21
	fi

	# check that mac is unique or auto
	num=` cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/mac/ and not /auto/);' | wc -l`
	numu=`cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/mac/ and not /auto/);' | sort -u | wc -l`
	if [ $num -ne $numu ]; then
		[ "$VERBOSE" = no ] || echo $"mac should be unique"
		echo_failure $"Checking configuration file:"
		abort 22
	fi

	# check that name is unique or auto
	num=` cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/name/ and not /auto/);' | wc -l`
	numu=`cat $config | grep -vE "^$|^#" | tr -s ' ' '\n' | perl -n -e 'print if (/name/ and not /auto/);' | sort -u | wc -l`
	if [ $num -ne $numu ]; then
		[ "$VERBOSE" = no ] || echo $"name should be unique"
		echo_failure $"Checking configuration file:"
		abort 23
	fi

	i=0
	bx_format=''
	cat $config | grep -vE "^$|^#" | grep -v $bprog 2> /dev/null | (
	while read str
	do
		let i=i+1
		set -- $str

		if [ $# -lt "$min_num_of_params" ] || [ $# -gt "$max_num_of_params" ]; then
			[ "$VERBOSE" = no ] || echo $"Invalid configuration in line number: $i"
			abort 28
		fi

		num_of_params=$#

		# set parameters
		rm -f $TMP/$$.tmp
		for (( i = 1; i <= $num_of_params; i++ ))
		do
			eval echo export \$$i >> $TMP/$$.tmp
		done
		eval `cat $TMP/$$.tmp`
		rm -f $TMP/$$.tmp

		check_value $i name    $name	|| abort 10
		check_value $i mac     $mac	|| abort 11
		check_value $i ib_port $ib_port	|| abort 12
		check_value $i vid     $vid	|| abort 13
		check_value $i vnic_id $vnic_id	|| abort 14
		check_value $i bx      $bx	|| abort 15
		check_value $i eport   $eport	|| abort 16
		check_value $i pkey    $pkey    || abort 17

		if ! [ -z "$emac" ]; then
			check_value $i mac   $emac   || abort 18
			check_value $i ip    $ip     || abort 19
		fi

		# make sure that BX format is the same on all lines (either all guid, all or names)
		# this avoids problems to sync between same GW that is represneted in different ways
		if [ -z `echo $bx | sed -n "/^\([0-9a-fA-F][0-9a-fA-F]:\)\{7\}[0-9a-fA-F][0-9a-fA-F]$/p"` ]; then
			_bx_format=guid
		else
			_bx_format=name
		fi

		if [ "$bx_format" != '' ] && [ "$bx_format" != "$_bx_format" ]; then
			[ $VERBOSE = no ] || echo "ERROR: BX format mismatch between config lines"
			abort 6
		else
			bx_format=$_bx_format
		fi
	done
	)

	if [ $? -ne 0 ]; then
		echo_failure $"Checking configuration file:"
		abort 7
	fi

	echo_success $"Checking configuration file:"
}

case "$1" in
	start)
		# check config file
		if [[ -z "$2" || "$2" != skip_check ]]; then
			check_config
		fi

		# start services
		prog_line=`cat $config | grep -vE "^$|^#" | grep ^$bprog | head -n1 | sed 's/^[ \t]*//'`
		prog_do=`echo $prog_line | awk '{print $1}' | cut -d= -f2`
		prog_param=`echo $prog_line | sed -e 's/'$bprog=$prog_do'//g'`
		[ $VERBOSE = no ] || echo "INFO: prog_line=$prog_line"
		[ $VERBOSE = no ] || echo "INFO: prog_do=$prog_do"
		[ $VERBOSE = no ] || echo "INFO: prog_param=$prog_param"

		if [ "$prog_do" == "yes" ]; then
			pid=`pidof -x -s $prog`
			if ! [ -z "$pid" ] ; then
				echo_warning $"$bprog is already running"
			else
				$prog $prog_param > /dev/null 2>&1 &
				pid=`pidof -x -s $prog`
				[ $VERBOSE = no ] || echo "INFO: prog cmd: $prog $prog_param"
				if ! [ -z "$pid" ] ; then
					echo_success $"Starting $bprog (pid $pid):"
				else
					echo_failure $"Starting $bprog:"
				fi
			fi
		fi

		# if a host admin parent vnics already exist, print a warning
		for cmd in `ls /sys/module/mlx4_vnic/vnic*-cmd* &> /dev/null && ls /sys/module/mlx4_vnic/vnic*-cmd*`
		do
                        _cmd=`test -r $cmd && cat $cmd`
                        _cmd=`echo $_cmd | sed 's/ib_port=[a-zA-Z0-9_:]*//g'`
			export $_cmd

			if ! ( grep -wq mac=$mac /sys/module/mlx4_vnic/vnic*-cmd ); then
				echo_warning $"vNic already configured, use \"`basename $0` reload\""
				abort 29
			fi
		done

		# Create new vNics
		i=0
		failed=0
		invalid=0

		cat $config | grep -vE "^$|^#" | grep -v $bprog 2> /dev/null | (
		while read str
		do
			let i=i+1
			set -- $str
			num_of_params=$#

			# set parameters
			rm -f $TMP/$$.tmp
			for (( j = 1; j <= $num_of_params; j++ ))
			do
				eval echo export \$$j >> $TMP/$$.tmp
			done

			name=$na_str mac=$na_str vnic_id=$na_str vid=$na_str  bxname=$na_str bxguid=$na_str eport=$na_str ipv4=$na_str ipv6=$na_str emac=$na_str pkey=$na_str parent=$na_str

			eval `cat $TMP/$$.tmp`
			rm -f $TMP/$$.tmp

			# Set bx format
			if [ -z `echo $bx | sed -n "/^\([0-9a-fA-F][0-9a-fA-F]:\)\{7\}[0-9a-fA-F][0-9a-fA-F]$/p"` ]; then
				bxname=$bx
			else
				bxguid=$bx
			fi

			if [ ! -z "$emac" ] && [ ! -z "$ip" ]; then
				ipv4=$ip
				emac=$emac
			fi

			# redefine "auto"
			if [ "$mac" = auto ]; then
				mac=$(get_random_mac)
			fi
			
			if [ "$name" = auto ]; then
				name=$(get_random_eth_name)
			fi

			if [ "$vnic_id" = auto ]; then
				vnic_id=$(get_random_vnic_id)
			fi

			# check IB port
			vnic_add=/sys/module/mlx4_vnic/host_add_${ib_port/:/_}
			if [ ! -f ${vnic_add} ]; then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid sysfs file ${vnic_add}"
				continue
			fi

			# check if it can be created
			system_macs=`/sbin/ip link show | awk '/ether/ {print $2}' 2>&1`
			if ( echo $system_macs | grep -iq $mac ); then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid configuration (mac $mac)"
				continue
			fi

			system_names=`/sbin/ip link show | grep ^[0-9] | awk  '{print $2}' | tr -d : 2>&1`
			if ( echo $system_names | grep -ixq $name ); then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid configuration (name $name)"
				continue
			fi

			# test that no child vNic already has this vNic name 
			if ( grep name=${name} /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ); then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid configuration (hadmin name $name)"
				continue
			fi

			# create command
			line="name=$name mac=$mac vnic_id=$vnic_id vid=$vid  bxname=$bxname bxguid=$bxguid eport=$eport ipv4=$ipv4 ipv6=$ipv6 emac=$emac pkey=$pkey parent=$parent"
			[ $VERBOSE = no ] || echo "INFO: cmd ${line} > ${vnic_add}"
			echo ${line} > ${vnic_add}

			tm=0
			vnic_id=$(($vnic_id))
			while ! ( grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ) && [ $tm -lt $add_timeout ]
			do
				sleep 1
				let tm=$tm+1
			done

			if ( grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ); then
				let created=$created+1
				/sbin/ifup $name > /dev/null
			else
				[ $VERBOSE = no ] || echo "ERROR: Failed to create vnic_id=$vnic_id port=${ib_port}"
				let failed=$failed+1
			fi
		done

		if [ $created -gt 0 ]; then
			echo_success $"Creating $created new vNics:"
		fi
		if [ $failed -gt 0 ]; then
			echo_failure $"Creating $failed new vNics:"
		fi
		if [ $invalid -gt 0 ]; then
			echo_failure $"Creating $invalid new vNics (invalid configuration):"
		fi
		)
	;;
	stop)
		# stop services
		pid=`pidof -x -s $prog`
		if ! [ -z "$pid" ] ; then
			if ( kill $pid &> /dev/null ); then
				echo_success $"Stopping $bprog:"
			else
				echo_failure $"Failed to stop $bprog"
			fi
		fi

		# destroy host admin vNics
		for vnic_cmd in /sys/module/mlx4_vnic/vnic*-cmd
		do
			[ $VERBOSE = no ] || echo "INFO: vnic_cmd =$vnic_cmd"
			test -r ${vnic_cmd} || continue

			_ib_port=`cat ${vnic_cmd} | egrep  "ib_port=[a-zA-Z0-9_:]*" -o`
			ib_port=`echo $_ib_port | cut -d= -f2 | tr : _`
			_cmd=`test -r $vnic_cmd && cat $vnic_cmd`

			if [ "$_cmd" == "" ]; then
				[ $VERBOSE = no ] || echo "INFO: empty file"
				continue
			fi

			vnic_del=/sys/module/mlx4_vnic/host_del_${ib_port}
			[ $VERBOSE = no ] || echo "INFO: vnic_del =$vnic_del"
			if [ ! -f ${vnic_del} ]; then
				[ $VERBOSE = no ] || echo "ERROR: could not find ${vnic_del}"
				let invalid=$invalid+1
				continue
			fi

			if [ ! -f ${vnic_cmd} ]; then
				[ $VERBOSE = no ] || echo "ERROR: could not find ${vnic_cmd}"
				let invalid=$invalid+1
				continue
			fi

			if [ "$_cmd" == "" ]; then
				continue
			fi

			cmd=`echo $_cmd | sed 's/ib_port=[a-zA-Z0-9_:]*//g'`
			[ $VERBOSE = no ] || echo "INFO: cmd      =$cmd"

			export $cmd

			# skip child vnics
			if [ $parent != $na_str ]; then
				[ $VERBOSE = no ] || echo "INFO: skip vNics created by other sources"
				continue
			fi

			[ $VERBOSE = no ] || echo "INFO: cmd ${cmd} > ${vnic_del}"
			echo "${cmd}" > ${vnic_del}

			tm=0
			cmd_fn="/sys/module/mlx4_vnic/vnic*-cmd"
			vnic_id=$(($vnic_id))

			while ( grep -w "vnic_id=$vnic_id" $cmd_fn 2>&1 | grep -w "$_ib_port" > /dev/null 2>&1 ) && [ $tm -lt $del_timeout ]
			do
				sleep 1
				let tm=$tm+1
			done

			if ( grep -w "vnic_id=$vnic_id" $cmd_fn 2>&1 | grep -w "$_ib_port" > /dev/null 2>&1 ); then
				[ $VERBOSE = no ] || echo "ERROR: Failed to destroy vnic_id=$vnic_id port=${ib_port}"
				let failed=$failed+1
			else
				let deleted=$deleted+1
			fi
		done

		if [ $deleted -gt 0 ]; then
			echo_success $"Destroying $deleted vNics:"
		fi
		if [ $failed -gt 0 ]; then
			echo_failure $"Destroying $failed vNics:"
		fi
		if [ $invalid -gt 0 ]; then
			echo_warning $"Destroying $invalid vNics (invalid sysfs files):"
		fi
	;;
	reload)
		if [[ -z "$2" || "$2" != skip_check ]]; then
			check_config
		fi

		# restart services (stop it if running, then start it if requested)
		prog_line=`cat $config | grep -vE "^$|^#" | grep ^$bprog | head -n1 | sed 's/^[ \t]*//'`
		prog_do=`echo $prog_line | awk '{print $1}' | cut -d= -f2`
		prog_param=`echo $prog_line | sed -e 's/'$bprog=$prog_do'//g'`
		[ $VERBOSE = no ] || echo "INFO: prog_line=$prog_line"
		[ $VERBOSE = no ] || echo "INFO: prog_do=$prog_do"
		[ $VERBOSE = no ] || echo "INFO: prog_param=$prog_param"

		pid=`pidof -x -s $prog`
		if ! [ -z "$pid" ] ; then
			if ( kill $pid &> /dev/null ); then
				echo_success $"Stopping $bprog:"
			else
				echo_failure $"Failed to stop $bprog:"
			fi
		fi

		pid=`pidof -x -s $prog`
		if [ "$prog_do" == "yes" ] && [ -z "$pid" ]; then
			$prog $prog_param > /dev/null 2>&1 &
			sleep 1
			pid=`pidof -x -s $prog`
			[ $VERBOSE = no ] || echo "INFO: prog cmd: $prog $prog_param"
			if ! [ -z "$pid" ] ; then
				echo_success $"Starting $bprog (pid $pid):"
			else
				echo_failure $"Starting $bprog:"
			fi
		fi

		# Remove devices that not in config
		for vnic_cmd in /sys/module/mlx4_vnic/vnic*-cmd
		do
			test -r $vnic_cmd || continue
                        _ib_port=`cat ${vnic_cmd} | egrep  "ib_port=[a-zA-Z0-9_:]*" -o`
                        ib_port=`echo $_ib_port | cut -d= -f2 | tr : _`
			vnic_del=/sys/module/mlx4_vnic/host_del_${ib_port}
			if [ ! -f ${vnic_del} ]; then
				[ $VERBOSE = no ] || echo "ERROR: no $vnic_del"
				let invalid=$invalid+1
				continue
			fi

			_cmd=`test -r $vnic_cmd && cat $vnic_cmd`
			cmd=`echo $_cmd | sed 's/ib_port=[a-zA-Z0-9_:]*//g'`

			[ $VERBOSE = no ] || echo "INFO: vnic_cmd =$vnic_cmd"
			[ $VERBOSE = no ] || echo "INFO: vnic_del =$vnic_del"
			[ $VERBOSE = no ] || echo "INFO:      cmd =$cmd"


			if [ "$cmd" == "" ]; then
				continue
			fi

			export $cmd

			# skip child vnics
			if [ $parent != $na_str ]; then
				[ $VERBOSE = no ] || echo "INFO: skip vNics created by other sources"
				continue
			fi

			found=1
			for token in $_cmd
			do
				[ $VERBOSE = no ] || echo "INFO: token=$token"
				opt=`echo $token | cut -d= -f1`
				val=`echo $token | cut -d= -f2`

				# adjust bxguid/bxname token
				if [ "$opt" == "bxname" ] || [ "$opt" == "bxguid" ]; then
					[ $VERBOSE = no ] || echo "INFO: adjust bxguid/bxname"
					token="bx=$val"
				fi

				# adjust IP token
				if [ "$opt" == "ipv4" ] || [ "$opt" == "ipv6" ]; then
					[ $VERBOSE = no ] || echo "INFO: adjust ipv4"
					token="$ip=$val"
				fi

				# if val is NA and the conf file has the same opt with non-NA => found=0
				if [ "$val" = "$na_str" ]; then
					if ( grep -vE "^$|^#" $config | grep -iw vnic_id=$vnic_id | grep $opt=.* | grep -v $na_str -q); then
						[ $VERBOSE = no ] || echo "INFO: no NA $token in $config"
						found=0
						break
					fi
				else
				# if conf line with same vnic_id don't have the token => found=0
					if ! ( grep -vE "^$|^#" $config | grep -iw vnic_id=$vnic_id | grep -iw $token -q); then
						[ $VERBOSE = no ] || echo "INFO: no $token in $config"
						found=0
						break
					fi
				fi
			done

			[ $VERBOSE = no ] || echo "INFO: found=$found"
			if [ "$found" == "0" ]; then
				[ $VERBOSE = no ] || echo "INFO: vnic found but not configured, delete it"
				[ $VERBOSE = no ] || echo "INFO: cmd ${cmd} > ${vnic_del}"
				echo "${cmd}" > $vnic_del

				tm=0
				vnic_id=$(($vnic_id))
				cmd_fn=/sys/module/mlx4_vnic/vnic*-cmd
				while ( grep -w "vnic_id=$vnic_id" $cmd_fn 2>&1 | grep -w "$_ib_port" > /dev/null 2>&1 ) && [ $tm -lt $del_timeout ]
				do
					sleep 1
					let tm=$tm+1
				done

				if ( grep -w "vnic_id=$vnic_id" $cmd_fn 2>&1 | grep -w "$_ib_port" > /dev/null 2>&1 ); then
					[ $VERBOSE = no ] || echo "ERROR: Failed to destroy vnic_id=$vnic_id port=${ib_port}"
					let failed=$failed+1
				else
					let deleted=$deleted+1
				fi
			fi
		done

		if [ $deleted -gt 0 ]; then
			echo_success $"Destroying $deleted vNics:"
		fi
		if [ $failed -gt 0 ]; then
			echo_failure $"Destroying $failed vNics:"
		fi
		if [ $invalid -gt 0 ]; then
			echo_warning $"Destroying $invalid vNics (invalid sysfs files):"
		fi

		# Create new devices
		i=0
		failed=0
		invalid=0

		cat $config | grep -vE "^$|^#" | grep -v $bprog 2> /dev/null | (
		while read str
		do
			let i=i+1
			set -- $str

			num_of_params=$#

			# set parameters
			rm -f $TMP/$$.tmp
			for (( j = 1; j <= $num_of_params; j++ ))
			do
				eval echo export \$$j >> $TMP/$$.tmp
			done

			name=$na_str mac=$na_str vnic_id=$na_str vid=$na_str  bxname=$na_str bxguid=$na_str eport=$na_str ipv4=$na_str ipv6=$na_str emac=$na_str pkey=$na_str parent=$na_str
			eval `cat $TMP/$$.tmp`
			rm -f $TMP/$$.tmp

			# Set bx format
			if [ -z `echo $bx | sed -n "/^\([0-9a-fA-F][0-9a-fA-F]:\)\{7\}[0-9a-fA-F][0-9a-fA-F]$/p"` ]; then
				bxname=$bx
			else
				bxguid=$bx
			fi

			if [ ! -z "$emac" ] && [ ! -z "$ip" ]; then
				ipv4=$ip
				emac=$emac
			fi

			# redefine "auto"
			if [ "$mac" = auto ]; then
				mac=$(get_random_mac)
			fi
			
			if [ "$name" = auto ]; then
				name=$(get_random_eth_name)
			fi

			if [ "$vnic_id" = auto ]; then
				vnic_id=$(get_random_vnic_id)
			fi

			# check IB port
			vnic_add=/sys/module/mlx4_vnic/host_add_${ib_port/:/_}
			if [ ! -f ${vnic_add} ]; then
				let invalid=$invalid+1
				continue
			fi

			if ( grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ); then
				# The device already exist
				output=`grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd`
				[ $VERBOSE = no ] || echo "INFO: The device already exist"
				[ $VERBOSE = no ] || echo "INFO: $output"
				continue
			fi

			line="name=$name mac=$mac vnic_id=$vnic_id vid=$vid  bxname=$bxname bxguid=$bxguid eport=$eport ipv4=$ipv4 ipv6=$ipv6 emac=$emac pkey=$pkey parent=$parent"
			[ $VERBOSE = no ] || echo "INFO: cmd ${line} > ${vnic_add}"
			# check if it can be created
			system_macs=`/sbin/ip link show | awk '/ether/ {print $2}' 2>&1`
			if ( echo $system_macs | grep -iq $mac ); then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid configuration (mac $mac)"
				continue
			fi

			system_names=`/sbin/ip link show | grep ^[0-9] | awk  '{print $2}' | tr -d : 2>&1`
			if ( echo $system_names | grep -ixq $name ); then
				let invalid=$invalid+1
				[ "$VERBOSE" = no ] || echo $"Invalid configuration (name $name)"
				continue
			fi

			# create command
			echo "$line" > ${vnic_add}

			tm=0
			vnic_id=$(($vnic_id))
			while ! ( grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ) && [ $tm -lt $add_timeout ]
			do
				sleep 1
				let tm=$tm+1
			done

			if ( grep -w "vnic_id=$vnic_id" /sys/module/mlx4_vnic/vnic*-cmd > /dev/null 2>&1 ); then
				let created=$created+1
			else
				[ $VERBOSE = no ] || echo "ERROR: Failed to create vnic_id=$vnic_id port=${ib_port}"
				let failed=$failed+1
			fi
		done

		if [ $created -gt 0 ]; then
			echo_success $"Creating $created new vNics:"
		fi
		if [ $failed -gt 0 ]; then
			echo_failure $"Creating $failed new vNics:"
		fi
		if [ $invalid -gt 0 ]; then
			echo_failure $"Creating $invalid new vNics (invalid configuration):"
		fi
		)
	;;
	restart)
		$0 stop skip_check
		$0 start
	;;
	status)
		# report services
		if ( cat $config | grep -vE "^$|^#" | grep ^$bprog &> /dev/null ); then
			pid_list=`pidof -x $prog`
			pid_num=`echo $pid_list | grep -vE ^$ | tr -s ' ' '\n' | wc -l`
			if [ "$pid_num" == "0" ]; then
				echo $"Services: $bprog is stopped."
			elif [ "$pid_num" == "1" ]; then
				echo $"Services: $bprog (pid $pid_list) is running..."
			else
				echo_warning $"Services: $bprog (pid $pid_list) are running..."
			fi
		fi

		# report vNics
		if ( ls /sys/module/mlx4_vnic/vnic*-cmd* &> /dev/null ) && ( ls /sys/module/mlx4_vnic/vnic*-cmd* &> /dev/null ); then
			echo $"Host Administrated vNics:"
			i=0
			for vnic_cmd in /sys/module/mlx4_vnic/vnic*-cmd
			do
				let i=i+1
				test -r $vnic_cmd || continue
				ib_port=`cat ${vnic_cmd} | egrep  "ib_port=[a-zA-Z0-9_:]*" -o | cut -d= -f2 | tr : _`
				cmd=`test -r $vnic_cmd && cat $vnic_cmd`
				[ $VERBOSE = no ] || echo "INFO: vnic_cmd =$vnic_cmd"
				[ $VERBOSE = no ] || echo "INFO:      cmd =$cmd"
				if [ "$cmd" == "" ]; then
					continue
				fi
				echo $"[$i] $cmd"
			done
		else
			echo $"Host Administrated vNics: none."
		fi
	;;
	*)
	echo $"Usage: $0 {start|stop|restart|reload|status}"
	abort 9
	;;
esac

abort 0

