#!/bin/bash
#
#
### BEGIN INIT INFO
# Provides:			mst
# Required-Start:		$local_fs 
# Should-Start:			
# Required-Stop:		
# Should-Stop:			
# Default-Start:	     2 3 4 5	
# Default-Stop:			 0 1 6
# Short-Description:		mst
# Description:			Starts and stops mst service from Mellanox tools package
### END INIT INFO

#insmod_flags="-f"
prefix="/usr/mst"
modprobe=/sbin/modprobe
is_bmc_switch=0
lspci=lspci

if [ `id -u` -ne 0 ]; then
    echo "-E- You must be root to use mst tool"
    exit 1
fi

if [ `uname -m` == "ppc" ]; then
    if [ -f /var/lib/chassis_info ]; then
        if [[ `cat /var/lib/chassis_info | grep CMC_TYPE` == *"BMC"* ]]; then
            is_bmc_switch=1
            lspci="/usr/sbin/lspci"
        fi
    fi
fi

if [ $is_bmc_switch == 0 ]; then
    if ${modprobe} -c | grep -q '^allow_unsupported_modules  *0'; then
        modprobe="${modprobe} --allow-unsupported-modules"
    fi
fi

if [ `uname -m` == "ppc" ]; then
        ENABLE_I2C_DEV=1
fi

if [[ `uname -a` == *MELLANOX* ]]; then
    IS_MLNXOS=1
fi

# Source function library.
action() {
    STRING=$1
    echo -n "$STRING"
    shift
    $*
    rc=$?
    if test $rc -ne 0
    then
        echo " - Failure: $rc"
        RETVAL=1
    else
        echo " - Success"
    fi
    return $rc
}
echo_success() {
    echo " - Success"
}
echo_failure() {
    echo " - Failure: $?"
}
failure() {
    echo -n "$* - Failure"
}

 
RETVAL=0

WITH_MSIX="with_msix"
WITH_UNKNOWN_ID="with_unknown"
WITH_I2CM="with_i2cm"
WITH_DEVI2C="with_i2cdev"
WITH_DEVLPC="with_lpcdev"
WITH_CABLES="with_cables"
WITH_FPGA="with_fpga"
FORCE_STOP="force"
ConnectX3_HW_ID="01F5"
ConnectX3_PRO_HW_ID="01F7"
SwitchX_HW_ID="0245"
SwitchIB_HW_ID="0247"
Spectrum_HW_ID="0249"
ConnectIB_HW_ID="01FF"
ConnectX4_HW_ID="0209"
ConnectX4LX_HW_ID="020B"
ConnectX5_HW_ID="020D"
Quantum_HW_ID="024D"

FPGA_VEN_ID="15b3"
FPGA_DEV_ID="600"

MST_START_FLAGS="[--$WITH_MSIX] [--$WITH_UNKNOWN_ID] [--$WITH_DEVI2C] [--$WITH_DEVLPC] [--$WITH_FPGA]" #[--$WITH_CABLES]" # [--$WITH_I2CM]  was removed to hide the feature
MST_START_HIDDEN_FLAGS="[--$WITH_I2CM]"
PMTUSB_NAME="Pmtusb"

# Mellanox dev directory
mdir="/dev/mst"        # Directory where MST devices created
mbindir=/usr/bin # Binary directory where MST-utils and modules are located
#@POST_MST_BIN_DIR@  # Update the bin dir by the post install script.

mlibdir=/usr/lib64 # Libraries directory where MST libs and modules are located
#@POST_MST_LIB_DIR@  # Update the lib dir by the post install script.
BASH_VERSION_LIB_PATH=${mlibdir}/mft/bash_libs/tools_version.sh

UNKNOWN_ID="UNKNWON_ID"

MST_CONF=/etc/mft/mst.conf
CONF_DIR=/etc/mft

#if [ `uname -m` == "ppc" ]; then
#    pcidir="/var/mst_pci"
#else
#    pcidir="$prefix/etc/pci" # Directory where PCI info should be saved
#fi
# Use the var to save the pci slot files
pcidir="/var/mst_pci"

# Default permission
perm="600"

# Old devices
devid_infiniscale=a87c     # MT21108

# Vendor / Device IDs
venid="15b3"         # Mellanox vendor ID

devid_pcurom="5a50"  # MT23108 PCIROM device ID


#    DevId              RstAddr   Description
dev_id_database=(\
    "6340               0xf0010   MT25408 [ConnectX VPI - IB SDR / 10GigE]"
    "634a               0xf0010   MT25418 [ConnectX VPI PCIe 2.0 2.5GT/s - IB DDR / 10GigE]"
    "6368               0xf0010   MT25448 [ConnectX EN 10GigE, PCIe 2.0 2.5GT/s]"
    "6372               0xf0010   MT25458 [ConnectX EN 10GigE 10GBaseT, PCIe 2.0 2.5GT/s]"
    "6732               0xf0010   MT26418 [ConnectX VPI PCIe 2.0 5GT/s - IB DDR / 10GigE]"
    "673c               0xf0010   MT26428 [ConnectX VPI PCIe 2.0 5GT/s - IB QDR / 10GigE]"
    "6750               0xf0010   MT26448 [ConnectX EN 10GigE, PCIe 2.0 5GT/s]"
    "675a               0xf0010   MT26458 [ConnectX EN 10GigE 10GBaseT, PCIe Gen2 5GT/s]"
    "676e               0xf0010   MT26478 [ConnectX EN 10GigE, PCIe 2.0 5GT/s]"
    "6746               0xf0010   MT26438 [ConnectX-2 VPI w/ Virtualization+]"
    "6764               0xf0010   MT26468 [Mountain top]"
    "bd34               0xf0010   IS4 IB SDR"
    "bd35               0xf0010   IS4 IB DDR"
    "bd36               0xf0010   IS4 IB QDR"
    "c738               0xf0010   SwitchX - 36 40G/64 10Gports InfiniBand and Ethernet Layer 2/3+ Switch"
    "cb20               0xf0010   Switch-IB"
    "1003               0xf0010   MT27500 [ConnectX-3]"
    "1005               0xf0010   MT27510 Family" 
    "1007               0xf0010   MT27520 ConnectX-3 Pro Family"
    "1009               0xf0010   MT27530 Family"
    "100b               0xf0010   MT27540 Family"
    "100d               0xf0010   MT27550 Family"
    "100f               0xf0010   MT27560 Family"
    "1011               0xf0010   MT27600 [Connect-IB ]"
    "1013               0xf0010   MT27620 [ConnectX-4]"
    "1015               0xf0010   MT27630 Family [ConnectX-4LX]"
    "1017               0xf0010   MT27800 Family [ConnectX-5]"
    "1019               0xf0010   MT28800 Family [ConnectX-5, Ex]"
    "101b               0xf0010   MT27660 Family"
    "cb84               0xf0010   Spectrum"
    "cf08               0xf0010   Switch-IB 2"
)

live_fish_id_database=(\
    "0191  0xf0010 MT25408 [ConnectX IB SDR Flash Recovery"
    "0246  0xf0010 SwitchX Flash recovery mode"
    "0249  0xf0010 Spectrum Flash recovery mode"
    "024b  0xf0010 Switch-IB 2 Flash recovery mode"
    "01F6  0xf0010 MT27500 [ConnectX-3 Flash Recovery]"
    "01F8  0xf0010 MT27500 [ConnectX-3 Pro Flash Recovery]"
    "01FF  0xf0010 MT27600 [Connect-IB Flash Recovery]"
    "0247  0xf0010 Switch-IB Flash recovery mode"
    "0209  0xf0010 MT27700 [ConnectX-4 Flash Recovery]"
    "020b  0xf0010 MT27630 [ConnectX-4LX Flash Recovery]"
    "020d  0xf0010 MT27800 [ConnectX-5 Flash Recovery]"
)


# Title
prog="MST (Mellanox Software Tools) driver set"



PATH=${PATH}:/sbin:/usr/bin:/bin:${mbindir}

kver=`uname -r`

if [ -r /etc/mst.conf ]; then
. /etc/mst.conf
fi

###
### MAP OPS (Assuming that the MAP is an array 
### with values in the format "<KEY>=<VAL>"
### -------
###

#declare -A devnums
#declare -A pf0devids

devnums=()
pf0devids=()

map_set() {

    map=$1
    key=$2
    val=$3
    if [ "$map" == "devnums" ]; then
        map_len=${#devnums[@]}
        for (( i=0; i<${map_len}; i++ )); do
            iKEY=${devnums[i]%%=*}
            if [ "$key" == "$iKEY" ]; then
                devnums[i]="$key=$val"
                return
            fi
        done
        devnums+=("$key=$val")
    elif [ "$map" == "pf0devids" ]; then
        map_len=${#pf0devids[@]}
        for (( i=0; i<${map_len}; i++ )); do
            iKEY=${pf0devids[i]%%=*}
            if [ "$key" == "$iKEY" ]; then
                pf0devids[i]="$key=$val"
                return
            fi
        done
        pf0devids+=("$key=$val")
    fi
}

map_get() {
    map=$1
    key=$2
    if [ "$map" == "devnums" ]; then
        for i in "${devnums[@]}"; do
            iKEY=${i%%=*}
            iVAL=${i#*=}
            if [ "$key" == "$iKEY" ]; then
                echo "$iVAL"
                return
            fi
        done
        printf "NA"
    elif [ "$map" == "pf0devids" ]; then
        for i in "${pf0devids[@]}"; do
            iKEY=${i%%=*}
            iVAL=${i#*=}
            if [ "$key" == "$iKEY" ]; then
                echo "$iVAL"
                return
            fi
        done
        printf "NA"
    fi
}

###
### PCI / PCICONF PCUROM
### --------------------
###

function is_device_bad() {
    dev=$1
    bar_info=`cat $dev | tail -1`
    bar_regexp="domain:bus:dev.fn=[[:xdigit:]]{4}:[[:xdigit:]]{2}:[[:xdigit:]]{2}.[[:xdigit:]]{1} bar=0x[[:xdigit:]]+ size=0x[[:xdigit:]]+"
    
    if [[ $bar_info =~ $bar_regexp ]]; then
        bar_size=`echo $bar_info | cut -d"=" -f4`
        if [ "$bar_size" == "0x0" ]; then
            return 1
        fi
    fi 
    return 0
}

### create group of PCI devices per one instance of InfiniHost (ddr,cr,uar)
### ------------------------------------------------------------------
create_pci_dev()
{
    ## For IA64 - do not create PCI devices
    #if [ "`uname -m`" = "ia64" ]; then
    #    return
    #fi

    local devname=$1
    local busdevfn=$2
    local devnum=$3
    local major=$4
    local minor=$5
#    local c_major=$6
#    local c_minor=$7
    local bar_step=$6
    shift 6
    #Get the fn from the BDF
    fn=$(echo ${busdevfn} | cut -f2 -d.)
    local bar=0 
    # PCI group of devices
    #     "_cr", "_uar"  and "_ddr" for InfiniHost
    #     ""     "_i2cm"            for Gamla
    for name
    do
        if [ x$name != xNOBAR ]; then
             new_dev=${mdir}/${devname}pci$name${devnum}
             if [[ "$fn" != "0" ]]; then
                new_dev=${new_dev}.${fn}
             fi
             if [ ! -c ${new_dev} ]; then
                 mknod -m ${perm} ${new_dev} c ${major} ${minor}
                 ${mbindir}/minit ${new_dev} $busdevfn $bar
                 if [ $? -ne 0 ]; then
                     rm ${new_dev}
                 fi
                 is_device_bad $new_dev; rc=$?
                 if [ $rc == "1" ]; then
                    ${mbindir}/mstop "$new_dev"
                    rm ${new_dev}
                  fi
             fi                  
             minor=$(( $minor + 1 ))
        fi
        bar=$(( $bar + $bar_step ))
    done
    # open mem access if closed.
    mem_byte=`setpci -s $busdevfn 4.B`
    let "mem_en = 0x$mem_byte & 0xf"
    if [ $mem_en -eq 0 ]; then
        let "mem_byte = 0x$mem_byte | 0x6"
        mem_byte=`printf "%x" $mem_byte`
        setpci -s $busdevfn 4.B=$mem_byte
    fi
    echo $minor
}

get_pciconf_dev_name()
{
    local devname=$1
    local devnum=$2    
    local fn=$3
    local name=$4
    local full_name="${mdir}/${devname}pciconf$name${devnum}"

    if [[ "$fn" != "0" ]]; then
        full_name=${full_name}.${fn}
    fi

    echo ${full_name}
}

### create a PCICONF device per one instance of infinihost
### -------------------------------------------------
create_pciconf_dev()
{
    local devname=$1
    local busdevfn=$2
    local devnum=$3
    local major=$4
    local minor=$5
    local name=$6

    #Get the fn from the BDF
    fn=$(echo ${busdevfn} | cut -f2 -d.)
    # PCICONF
    fullname=`get_pciconf_dev_name $devname $devnum $fn $name`
    if [ ! -c ${fullname} ]; then
        mknod -m ${perm} $fullname c ${major} ${minor}
        ${mbindir}/minit $fullname ${busdevfn} 88 92
        if [ $? -ne 0 ]; then
            rm ${mdir}/${devname}pciconf$name${minor}
        fi
    fi
    minor=$((  $minor + 1 ))
}

#write all-ones to PCI Vital Product Data(VPD) capability register
#wait till top bit read as 0
clear_VPD()
{
   busdevfn=$1
   reg_vpd=0
   stat=`setpci -s $busdevfn 06.B`
   if (($stat & 0x10 ));then #if there is a capabilities list
     reg=`setpci -s $busdevfn 34.L`     #get 1st capability reg
     while (( $reg != 0 )); do
       regval=0x`setpci -s $busdevfn ${reg}.L`
       if (($((regval & 0xff)) == 0x03)); then
         reg_vpd=$reg
         break;
       fi
       reg=${regval:6:2}
     done
     #echo reg_vpd=$reg_vpd

     if (($reg_vpd)); then
         setpci -s $busdevfn ${reg_vpd}.L=ffffffff
         regval=0x`setpci -s $busdevfn ${reg_vpd}.L`
         while (( $regval & 0x80000000 )); do
           regval=`setpci -s $busdevfn ${reg_vpd}.L`
         done
     fi
   fi
}

is_gamla_anfa()
{
    if [ "$1" == "$devid_infiniscale" ]; then
        echo 1
    else
        echo 0
    fi
}

get_pci_dev_args()
{
    local with_msix=$1
    local devid=$2
    local conf_dev=$3
    local dev_args=""
    
    # legacy devices
    is_gamla_anafa_ind=$(is_gamla_anfa $devid)
    if [ $is_gamla_anafa_ind -eq 1 ]; then
        dev_args=" 0 \"\" _i2cm"
    else
        # HCAs
        hw_dev_id=`get_dev_id_by_mst $conf_dev`
        dev_args="2 _cr"
    
        if [ x$hw_dev_id == x$ConnectIB_HW_ID ] || [ x$hw_dev_id == x$ConnectX4_HW_ID ] || [ x$hw_dev_id == x$ConnectX4LX_HW_ID ] || [ x$hw_dev_id == x$ConnectX5_HW_ID ]; then
            dev_args="2 NOBAR _cr"
        fi
        if [ "$with_msix" == "1" ]; then
            dev_args="$dev_args _msix"
        fi
    fi

    echo $dev_args
}

get_clear_vpd()
{
    local devid=$1
    is_gamla_anafa_ind=$(is_gamla_anfa $devid)
    if [ $is_gamla_anafa_ind -eq 1 ]; then
        echo 1
    else
        echo 0
    fi
}

get_conf_create_ind()
{
    local devid=$1
    is_gamla_anafa_ind=$(is_gamla_anfa $devid)
    if [ $is_gamla_anafa_ind -eq 1 ]; then
        echo 0
    else
        echo 1
    fi
}


get_dev_id()
{
    local oper=$1
    local devidarg=$2
    local devid_lspci_out=$3

    local devid=$devidarg

    if [ "$devidarg" == $UNKNOWN_ID ] ; then
        if [ "$oper" == "before" ]; then
            devid=""
        else
            devid=`echo $devid_lspci_out | cut -f2 -d"\""`
        fi
    fi
    echo $devid
}

get_dev_name()
{
    dev_id=$1
    dev_id_dec=`printf %d 0x$dev_id`
    devname="mt"$dev_id_dec"_"
    echo $devname
}

prepare_create_pci_dev()
{
    local devidarg=$1
    local pciminor=$2
    local pciconfminor=$3
    local pcimajor=$4
    local pciconfmajor=$5
    local with_msix=$6

    shift 6
    
    devid=$(get_dev_id "before" $devidarg)

    new_minors=`(
    echo $pciminor $pciconfminor
    ${lspci} -m -n -d ${venid}:$devid 2> /dev/null | while read str
    do
        set -- $str
        busdevfn=$1

        if [[ "$busdevfn" != *\:*\:*\.* ]]; then 
            busdevfn="0000:$busdevfn"
        fi
        devid=$(get_dev_id "after" $devidarg $4)

        devname=$(get_dev_name $devid)
        
        #Get the fn from the BDF
        fn=$(echo ${busdevfn} | cut -f2 -d.)
        busdev=$(echo $busdevfn | cut -d':' -f1-2 | sed 's/://g')
        if [[ "$fn" == "0" ]]; then
            #pf0devids[$busdev]=$devid
            map_set "pf0devids" ${busdev} ${devid}
        fi
        #pf0devid=${pf0devids[$busdev]}
        pf0devid=$(map_get "pf0devids" $busdev)
        
        val=$(map_get "devnums" $pf0devid)
        if [ "$val" == "NA" ]; then
            #devnums[$pf0devid]=0
            map_set "devnums" $pf0devid 0
        else
            if [[ "$fn" == "0" ]]; then
                #let devnums[$pf0devid]+=1
                new_val=$(expr $val + 1)
                map_set "devnums" $pf0devid $new_val;
            fi
        fi
        devnum=$(map_get "devnums" $pf0devid)
        #devnum=${devnums[$pf0devid]}
        conf_dev_name=$(get_pciconf_dev_name $devname $devnum $fn)
        
        needs_clear_vpd=$(get_clear_vpd $devid)
        needs_conf_create=$(get_conf_create_ind $devid)

        #write all-ones to PCI Vital Product Data(VPD) capability register
        if [ $needs_clear_vpd -ne 0 ]; then
            clear_VPD $busdevfn
        fi

        if [ $needs_conf_create -ne 0 ]; then
            create_pciconf_dev $devname $busdevfn $devnum $pciconfmajor $pciconfminor
            pciconfminor=$((  $pciconfminor + 1 ))
        fi

        
        pci_dev_args=$(get_pci_dev_args $with_msix $devid $conf_dev_name)
        next_pciminor=$(create_pci_dev $devname $busdevfn $devnum $pcimajor $pciminor $pci_dev_args)
        pciminor=$next_pciminor
        
        echo $pciminor $pciconfminor
    done
    )| tail -1`
    echo $new_minors
}


prepare_create_pci_dev_live_fish()
{
    local devidarg=$1
    local pciconfminor=$2
    local pciconfmajor=$3

    devnum=0

    devid=$(get_dev_id "before" $devidarg)
    
    new_pciconfminor=`(
    echo $pciconfminor
    ${lspci} -m -n -d ${venid}:${devid} 2> /dev/null | sort | while read str
    do
        set -- $str
        busdevfn=$1
        devid=$(get_dev_id "after" $devidarg $4)

        devname=$(get_dev_name $devid)
        create_pciconf_dev $devname $busdevfn $devnum $pciconfmajor $pciconfminor

        devnum=$((  $devnum + 1 ))
        pciconfminor=$((  $pciconfminor + 1 ))
        echo $pciconfminor
    done
    )| tail -1`
    echo $new_pciconfminor
}

find_mtusb_rename()
{
    declare -i i
    i=0

    for s in ${ids[@]}
    do
        if [ "0x$1" == ${s} ]; then
            break
        fi
        i=i+1
    done
    if [[ "${devices_names[$i]}" != "" ]]; then
        mtusb_name=${devices_names[$i]}
    else
        rc=1
        while test $rc -ne 0 ; do
            mtusb_name=${mst_usb_dev}mtusb-1
            mst_usb_dev=X$mst_usb_dev
            check_existence $mtusb_name "n"
            rc=$?
        done
    fi
        
}

generate_serial_file ()
{
    bus=$1
    dev=$2
    fname="usb_${bus}_${dev}_serial"
    serial_file_name=${CONF_DIR}/${fname}
    if [ ! -f $serial_file_name ]; then
        iserial=$(lsusb -v -s $bus:$dev 2> /dev/null | grep iSerial | awk '{print $3}')
        echo ${iserial} > ${serial_file_name}
    fi
}

create_mtusb_devices()
{

    # Create MTUSB devices
    if [ `uname -m` == "ppc" ]; then
        return
    fi

    local dimax_vend=0x0abf
    local dimax_prod=0x3370

    local mst_usb_dev=""

    OLD_IFS=$IFS
    IFS=$'\n';
    warn_msg="-W- Missing "lsusb" command, skipping MTUSB devices detection"
    if [ "$IS_MLNXOS" == "1" ]; then
        warn_msg=""
    fi
    command -v lsusb >/dev/null || { echo ${warn_msg}; return; }
    for lsusb_out in `lsusb -d $dimax_vend:$dimax_prod 2> /dev/null`;
    do
        IFS=$OLD_IFS
        found_devs=0
        local bus=$(echo $lsusb_out | cut -f2 -d" ")
        local device=$(echo $lsusb_out | cut -f1 -d":" | cut -f4 -d " ")
        for usb_dir in /dev/bus/usb /proc/bus/usb;
        do

            if ! test -d $usb_dir; then
                continue
            fi
            usb_dev="$usb_dir/$bus/$device"
            if chmod 0666 $usb_dev 2> /dev/null; then
                if ${mbindir}/dimax_init $usb_dev > /dev/null ; then
                    found_devs=1
                    if [[ "${ENABLE_RENAMING}" == "1" ]]; then
                         usb_serial=`get_mtusb_serial $bus $device`
                         find_mtusb_rename $usb_serial
                    else 
                        mtusb_name=${mst_usb_dev}mtusb-1
                        mst_usb_dev=X$mst_usb_dev
                    fi
                    action "MTUSB-1 USB to I2C Bridge" ln -fs $usb_dev ${mdir}/${mtusb_name} 2> /dev/null
                    generate_serial_file $bus $device
                else
                    diolan_mod=`lsmod | grep diolan`
                    if [[ "${diolan_mod}" == "" ]]; then
                        echo "Can't initialize MTUSB-1 USB to I2C Bridge"
                    else
                        #echo "Failed to initialize MTUSB-1, try to blacklist the module i2c-diolan-u2c"
                        echo "Failure to initialize MTUSB-1 due to being owned by i2c-diolan-u2c."
                        echo "To use MTUSB-1 device, please remove i2c-diolan-u2c module (Run: modprobe -r i2c-diolan-u2c)"
                        echo "Blacklisting the module i2c-diolan-u2c can be done by adding it to /etc/modprobe.d/blacklist."
                    fi
                fi   
            fi
            # If devices were found on first dir, don't search the other dir (may be doplicated)
            if [ "$found_devs" == "1" ]; then 
                break;
            fi
        done

    done
    return
}

check_lspci_existance()
{
    check_lspci=`${lspci} --version 2> /dev/null`
    if [ $? -ne 0 ]; then
        echo "-E- Could not find lspci, you may need to install \"pciutils\" package."
        return 1
    fi
}

create_pci_devices()
{
    local with_msix=$1
    local with_unknown=$2

    # ------------------------------------
    # Determine PCI/PCICONF major numbers.
    # Initialize PCI/PCICONF minor numbers.
    # ------------------------------------
    
    check_lspci_existance    
    if [ $? -ne 0 ]; then
        return 1
    fi
    mstr=`cat /proc/devices | grep 'mst_pci$'`
    if [ $? -ne 0 ]; then
        echo
        echo "mst_pci driver not found"
        return 1
    fi
    set -- $mstr
    pcimajor=$1
    mstr=`cat /proc/devices | grep 'mst_pciconf$'`
    if [ $? -ne 0 ]; then
        echo
        echo "mst_pciconf driver not found"
        return 1
    fi
    set -- $mstr
    pciconfmajor=$1
    pciminor=0
    pciconfminor=0


    if [ "$with_unknown" == "1" ]; then
        pci_pciconf_minor=$(prepare_create_pci_dev $UNKNOWN_ID $pciminor $pciconfminor $pcimajor $pciconfmajor $with_msix)
        set -- $pci_pciconf_minor
        pciminor=$1
        pciconfminor=$2
    else
        element_count=${#dev_id_database[@]}
        index=0
        while [ "$index" -lt "$element_count" ]; do
            set -- ${dev_id_database[$index]}
            devid=$1
            pci_pciconf_minor=$(prepare_create_pci_dev $devid $pciminor $pciconfminor $pcimajor $pciconfmajor $with_msix)
            set -- $pci_pciconf_minor
            pciminor=$1
            pciconfminor=$2

            ((index++))
        done
    fi

    element_count=${#live_fish_id_database[@]}
    index=0
    while [ "$index" -lt "$element_count" ]; do
        set -- ${live_fish_id_database[$index]}
        devid=$1
        pci_pciconf_minor=$(prepare_create_pci_dev_live_fish $devid $pciconfminor $pciconfmajor)
        pciconfminor=$pci_pciconf_minor
        ((index++))
    done
    return

}

# create all devices
create_devices()
{
    echo "$1"
    create_pci_devices $2 $3
    create_mtusb_devices
    return
}

is_module()
{
local RC

    /sbin/lsmod | grep -w "$1" > /dev/null 2>&1
    RC=$?

return $RC
}
load_module()
{
    mod_name=$1
    mod_file_path=$2
    load_cmd=$3
    load_cmd_string=$4
    
    if is_module ${mod_name}
    then 
        echo "[warn] ${mod_name} is already loaded, skipping"
    else 
            action "${load_cmd_string}" "${load_cmd}" "${load_cmd_flags}" "${mod_file_path}"        
    fi

}

get_dev_id_by_mst()
{
    mst_dev=$1
    dev_rev=`${mbindir}/mcra ${mst_dev} 0xf0014`
    local str=${dev_rev:6:4}
    str=`echo $str | tr a-z A-Z`
    echo $str   
}

# Function to check the conf file if it's good !

check_existence()
{
    obj=$1
    if [ $2 == "n" ]; then
        list=("${devices_names[@]}")
    else
        list=("${ids[@]}")
    fi
    for l in "${list[@]}"
    do
        #echo comparing $obj with $l
        if [ $obj == $l ]; then
            return 1
        fi
    done
    return 0
}

check_conf()
{
    if [ ! -f $MST_CONF ]; then
        echo "-W- Missing mst conf file: $MST_CONF"
        return 0
    fi
    conf=$1
    while read line
    do
        if [ ${#line} != 0 ]; then
            if [[ "$line" == "#"* ]]; then
                continue
            fi
            conf_opcode=$(echo ${line} | awk '{print $1}')
            if [[ ${conf_opcode} == "RENAME" ]]; then 
                dev_type=$(echo ${line} | awk '{print $2}')
                devname=$(echo ${line} | awk '{print $3}')
                id=$(echo ${line} | awk '{print $4}')
                if [[ ${dev_type} == "USB" ]]; then
                   devname=${devname}-mtusb-1
                else
                    echo -W- Renaming ${dev_type} devices is not supported !
                    continue
                fi
                check_existence ${devname} "n"
                rc=$?
                if test $rc -ne 0 ; then
                    echo -e "-E- The Conf file is not right !\n-E- line: $line have duplicated device name, Renaming was ignored !"
                    ENABLE_RENAMING=0
                    return 1
                fi
                check_existence ${id} "id"
                rc=$?
                if test $rc -ne 0 ; then
                    echo -e "$res-E- The Conf file is not right !\n-E- line: $line have duplicated ID, Renaming was ignored !"
                    ENABLE_RENAMING=0
                    return 1
                fi
                devices_names+=(${devname})
                ids+=(${id})
                ENABLE_RENAMING=1
            else
                echo -W- OPCODE=${conf_opcode} is not supported in line: $line !
                continue
            fi
        fi
    done <$MST_CONF
}
create_fpga_devices()
{
    check_lspci_existance
    if [ $? -ne 0 ]; then
        return 1
    fi
    lspci -D -d ${venid}:${FPGA_DEV_ID} | cut -f1 -d" " | while read str
    do
        touch $mdir/${str}_fpga
    done
}

# Main function
start()
{
    local with_msix=0
    local with_unknwon_id=0
    local with_i2cm=0

    check_conf $MST_CONF
    
    while (( "$#" )); do
        if [ "$1" == "--$WITH_MSIX" ]; then
            with_msix=1
        elif [ "$1" == "--$WITH_UNKNOWN_ID" ]; then
            with_unknwon_id=1
        elif [ "$1" == "--$WITH_I2CM" ]; then
            with_i2cm=1               
        elif [ "$1" == "--$WITH_DEVI2C" ]; then
            ENABLE_I2C_DEV=1
        elif [ "$1" == "--$WITH_DEVLPC" ]; then
            ENABLE_LPC_DEV=1
        elif [ "$1" == "--$WITH_FPGA" ]; then
            ENABLE_FPGA_DEV=1
        else
            echo "-E- Unknown argument $1"
            exit 1
        fi

        shift
    done



    echo "Starting $prog"

    # Create empty MST devices directory
    # rm -fr ${mdir}
    # mkdir ${mdir}
    if [ ! -d ${mdir} ]; then
        mkdir ${mdir}
    fi


    if [ "$ENABLE_I2C_DEV" == "1" ]; then
        if [ ! -r /dev/i2c-0 ]; then
            action "Loading I2C modules" "${modprobe}" i2c-dev
            sleep 0.1
        fi
        for f in /dev/i2c-[0-9]; do
                dev=`basename $f`
                I2C_DEV=${mdir}/dev-$dev
                rm -f $I2C_DEV
                ln -s $f $I2C_DEV
        done
    fi
    if [ "$ENABLE_LPC_DEV" == "1" ]; then
        if [ -f /sys/devices/platform/lpci2c/io_regions ]; then
            cat /sys/devices/platform/lpci2c/io_regions > ${mdir}/dev-lpc-1
        fi
    fi
    if [ "$ENABLE_FPGA_DEV" == "1" ]; then
        create_fpga_devices
    fi

    MST_PCI_MOD="mst_pci"
    MST_PCICONF_MOD="mst_pciconf"        
    load_module "${MST_PCI_MOD}"     "${MST_PCI_MOD}"     "${modprobe}"  "Loading MST PCI module"       
    load_module "${MST_PCICONF_MOD}" "${MST_PCICONF_MOD}" "${modprobe}"  "Loading MST PCI configuration module" 

    # create all related devices
    create_devices "Create devices"  $with_msix $with_unknwon_id
    if  ls ${mdir}| grep -e "_cr" > /dev/null 2>&1
    then :
    else
        action "Unloading MST PCI module (unused)" modprobe -r mst_pci
    fi
    if ls ${mdir}| grep -e "conf" > /dev/null 2>&1
    then :
    else
        action "Unloading MST PCI configuration module (unused)" modprobe -r mst_pciconf
    fi


    if [ ! -e /dev/i2c-0 ]; then
       if lsmod | grep "i2c_dev" > /dev/null ; then
          action "Unloading I2C module (unused)" modprobe -r i2c-dev
       fi
    fi
    dev_name="${PMTUSB_NAME}-1"
    if [ x"$with_i2cm" == x"1" ]; then 
        for dev in `ls ${mdir}| grep -e "conf"`; do
            conf_dev="${mdir}/$dev"
            dev_rev=`get_dev_id_by_mst $conf_dev`
            if [ x"${dev_rev}" == x"${ConnectX3_HW_ID}" ] || [ x"${dev_rev}" == x"${ConnectX3_PRO_HW_ID}" ]; then
                action "PCIe to I2c Bridge" ln -s ${conf_dev} ${mdir}/${dev_name}
                dev_name="P${dev_name}"
            fi
            # 
        done
    fi
    ###################################
    #  Newton FPGA (CX4LX) detection  #
    ###################################
    if [ "$ENABLE_FPGA_DEV" == "1" ]; then
        if [ -f ${mbindir}/mlx_fpga ]; then
            for f in /dev/mst/*;
            do
                if [[ $f =~ /dev/mst/mt4117_pciconf[0-9]+ ]]; then
                    mlx_fpga_out=`${mbindir}/mlx_fpga -d $f query`
                    if [[ $mlx_fpga_out == *"Device Type:         CX4LX_NEWTON"* ]]; then
                        touch "${f}_fpga"
                    fi
                fi
            done
        fi
        # mlx5 driver interface for FPGA
        OLD_IFS=$IFS
        IFS=' '
        # If driver present
        lsmod | grep "mlx_accel_tools" &> /dev/null
        if [ $? == 0 ] ; then
            suffix="_accel_tools"
            for f in /dev/mlx5_*_accel_tools
            do
                base_name=`basename $f`
                dev_name=${base_name%$suffix}
                pci_name=`echo $dev_name | cut -d "-" -f 2`
                devs_arr=(`grep -l domain:bus:dev.fn=$pci_name /dev/mst/mt*pciconf*[0-9]`)
                mst_dev_t=${devs_arr[0]}
                # set two links for I2C and RoCE
                if [ "${mst_dev_t}" != "" ]; then
                    ln -sf $f "${mst_dev_t}_fpga_i2c"
                    ln -sf $f "${mst_dev_t}_fpga_rdma"
                fi
            done
        fi
        IFS=$OLD_IFS
    fi
}

function check_module_busy()
{
    mod_name=$1
    used_by=$(lsmod | grep -w ${mod_name} | awk '{print $3}')
    if [ "${used_by}" == "0" -o "${used_by}" == "" ]; then
        return 0
    else
        echo "-E- ${mod_name} module is in use, stop operation failed, you may use \"mst stop --force\" to force stop operation."
        exit 1
    fi
}

clean_mdir()
{
    
    if [ "$IS_MLNXOS" == "1" ]; then
        find ${mdir}/* -not -name "*dev-i2c*" -exec rm -f {} \;
    else
        rm -fr ${mdir}
    fi
    rm -fr ${CONF_DIR}/usb*serial
}

stop()
{

    if [ "$1" == "--$FORCE_STOP" ]; then
        ENABLE_FORCE_STOP=1
    fi

    echo "Stopping $prog"

    serv_stop
    if [ "${ENABLE_FORCE_STOP}" != "1" ]; then
        check_module_busy mst_pciconf
        check_module_busy mst_pci
    fi
    
    for dev in ${mdir}/*
    do
        if [ -e "$dev" ]; then
          ${mbindir}/mstop "$dev" >/dev/null 2>&1
        fi
    done

    if  lsmod | grep mst_pciconf > /dev/null
    then
        action "Unloading MST PCI configuration module" modprobe -r  mst_pciconf
    fi
    if  lsmod | grep mst_pci | grep -v mst_pciconf> /dev/null
    then
        action "Unloading MST PCI module" modprobe -r mst_pci
    fi


    if lsmod | grep i2c_dev > /dev/null
    then
         action "Unloading i2c driver" modprobe -r i2c_dev
    fi
    
    clean_mdir
}

print_chip_rev_internal()
{
    export MTCR_REMOTE_WARN=1
    local dev_rev=`${mbindir}/mcra $1 0xF0014`
    unset MTCR_REMOTE_WARN
    if [ "$dev_rev" == "0xbad0cafe" ]; then
        local str="NA"
    else
        local str=${dev_rev:4:2}
        str=`echo $str | tr a-z A-Z`
    fi
    echo "                                   Chip revision is: $str"
}

print_chip_rev()
{
    local dev=$1
    if expr match "$dev" ".*_pci_cr" > /dev/null 2>&1
    then
        print_chip_rev_internal $dev
    fi
    if expr match "$dev" ".*_pciconf" > /dev/null 2>&1
    then
        print_chip_rev_internal $dev
    fi
}
get_mtusb_serial()
{
    bus=$1
    dev=$2
    fname="usb_${bus}_${dev}_serial"
    serial_file_name=${CONF_DIR}/${fname}
    if [ ! -f ${serial_file_name} ]; then
        generate_serial_file $1 $2
        if [ ! -f ${serial_file_name} ]; then
            echo ERROR
            return
        fi
    fi

    iserial=$(cat ${serial_file_name})
    if [ ${#iserial} == 0 ]; then
        echo ERROR
    else
        echo ${iserial}
    fi
}

print_mtusb_sn()
{
    bus=`readlink $1 | rev | cut -f2 -d/ | rev`
    dev=`readlink $1 | rev | cut -f1 -d/ | rev`
    iserial=`get_mtusb_serial $bus $dev`
    printf "%35siSerial = 0x$iserial\n"
}

function ignore_cr_devs()
{
    set dev=$1

    if expr match "$dev" ".*_pci_cr"  > /dev/null 2>&1; then
        if expr match $kver ".*460ex" > /dev/null 2>&1; then
            if [ `uname -m` == "ppc" ]; then
                echo "YES"
                return
            fi
        fi
    fi

    echo "NO"
}

function ignore_phys_fn()
{
    set dev=$1
    if expr match "$dev" ".*_pciconf[0-9]*\.[0-9]*"  > /dev/null 2>&1; then
        echo "YES"
        return
    fi
    if expr match "$dev" ".*_pci_cr[0-9]*\.[0-9]*"  > /dev/null 2>&1; then
        echo "YES"
        return
    fi
    echo "NO"
    
}
IGNORE_FUNCS=1

print_ul_mdevices_info()
{
    echo -e "\nPCI Devices:"
    echo -e "------------\n"
    mdevices="$(mdevices_info -vv)"
    RC=$?
    if [ $RC != 0 ]; then
        echo -e "\tNo devices were found.\n"
        return
    fi
    echo "${mdevices}" | while read -r line
    do
        if expr match "$line" "[-]*" > /dev/null 2>&1; then
            continue
        fi
        if expr match "$line" ".*PCI.*" > /dev/null 2>&1; then
            continue
        fi
        read -a arr <<< $line
        if [[ "$IGNORE_FUNCS" == "1" ]]; then
            if [[ "${arr[2]}" == *"1" ]]; then
                continue
            fi
        fi
        if [[ -z "${arr[2]}" ]]; then
            continue;
        fi
        echo -e "${arr[2]}\n"
    done
    
}

print_status()
{
    local verbose=$1
    is_mst_loaded=0
    
    # Check modules
    echo "MST modules:"
    echo "------------"
    if  is_module mst_pci
    then
        echo "    MST PCI module loaded"
        is_mst_loaded=1
    else
        echo "    MST PCI module is not loaded"
    fi
    if  is_module mst_pciconf
    then
        echo "    MST PCI configuration module loaded"
        is_mst_loaded=1
    else
        echo "    MST PCI configuration module is not loaded"
    fi
    
    if  cat /proc/devices | grep mst_ppc > /dev/null
    then
       echo  "    MST PPC Bus module loaded"
    fi

    if [ "$verbose" == "-v" -o "$verbose" == "-vv" ]; then
        mdevices_info $verbose
        return
    fi
    
    if [ $is_mst_loaded != 1 ]; then
        #echo "User Level devices:"
        #echo "-------------------"
        print_ul_mdevices_info
        if [ ! -d ${mdir} ]; then
            return
        fi
    fi
    # Devices
    devcnt=0
    rcnt=0
    rdevs=""
    ibcnt=0
    ibdevs=""
    lpccnt=0
    lpcdev=""
    fpgacnt=0
    fpgadev=""
    cabledevs=""
    cablescnt=0
    if [ $is_mst_loaded -eq 1 ]; then
        echo
        echo "MST devices:"
        echo "------------"
    fi
    for dev in ${mdir}/*
    do
        if [ -r "$dev" ]; then
            if expr match "$dev" ".*fpga.*" > /dev/null 2>&1
            then
                fpgadevs=$fpgadevs" $dev"
                fpgacnt=$((  $fpgacnt + 1 ))
            elif expr match "$dev" ".*cable.*" > /dev/null 2>&1
            then
                cabledevs=$cabledevs" $dev"
                cablescnt=$((  $cablescnt + 1 ))
            elif expr $dev : '.*:' > /dev/null 2>&1
            then
                rdevs=$rdevs" $dev"
                rcnt=$((  $rcnt + 1 ))
            elif expr match "$dev" ".*lid-0x[0-9A-Fa-f]*" > /dev/null 2>&1
            then
                ibdevs=$ibdevs" $dev"
                ibcnt=$((  $ibcnt + 1 ))
            elif expr match "$dev" ".*lid-[0-9]*" > /dev/null 2>&1
            then
                ibdevs=$ibdevs" $dev"
                ibcnt=$((  $ibcnt + 1 ))
            elif expr match "$dev" ".*ibdr-[0-9]*" > /dev/null 2>&1
            then
                ibdevs=$ibdevs" $dev"
                ibcnt=$((  $ibcnt + 1 ))
            elif expr match "$dev" ".*${PMTUSB_NAME}.*" > /dev/null 2>&1
            then
                    echo -e "$dev\t\t - PCIe to I2C adapter as I2C master"
            elif expr match "$dev" ".*usb.*" > /dev/null 2>&1
            then
                    #echo -e "$dev\t\t - USB to I2C adapter as I2C master"
                    printf "%-33s- USB to I2C adapter as I2C master\n" "$dev"
                    print_mtusb_sn $dev
            elif expr match "$dev" ".*lpc.*" > /dev/null 2>&1
            then
                lpcdevs=$lpcdevs" $dev"
                lpccnt=$((  $lpccnt + 1 ))
            elif expr match "$dev" ".*dev-i2c.*" > /dev/null 2>&1
            then
                    echo -e "$dev\t\t - Embedded I2C master"
            else
                ignore_dev=$(ignore_cr_devs $dev)
                if [ "$ignore_dev" == "NO" ]; then
                    ignore_dev=$(ignore_phys_fn $dev)
                fi
                if [ "$ignore_dev" == "NO" ]; then
                    cat $dev
                    if expr match "$dev" ".*_pci_cr" > /dev/null 2>&1
                    then
                        print_chip_rev $dev
                    fi
                    if expr match "$dev" ".*_pciconf" > /dev/null 2>&1
                    then
                        print_chip_rev $dev
                    fi
                fi
            fi
            devcnt=$((  $devcnt + 1 ))
       fi
    done
    
    if [ ${rcnt} -ne 0 ]; then
        echo
        echo "Remote MST devices:"
        echo "-------------------"
        for dev in $rdevs
        do
            echo $dev
            print_chip_rev $dev
        done
    fi

    if [ ${ibcnt} -ne 0 ]; then
        echo
        echo "Inband devices:"
        echo "-------------------"
        for dev in $ibdevs
        do
            echo $dev
        done
    fi
    
    if [ ${lpccnt} -ne 0 ]; then
        echo
        echo "LPC device:"
        echo "-------------------"
        for dev in $lpcdevs
        do
            echo $dev
        done
    fi
    
    if [ ${fpgacnt} -ne 0 ]; then
        echo
        echo "FPGA devices:"
        echo "-------------------"
        for dev in $fpgadevs
        do
            echo $dev
        done
    fi
    
    if [ ${cablescnt} -ne 0 ]; then
        echo
        echo "Cables:"
        echo "-------------------"
        for dev in $cabledevs
        do
            echo `basename $dev`
        done
    fi
}



# return the matching slot file name in the PCI fs
# Check if domain need to be added.
get_pci_file()
{
    slot=$1

    local prefix=/proc/bus/pci/

    if echo $slot | grep ":..:"  > /dev/null 2>&1
    then
        dombus=`echo $slot | cut -f1,2 -d:`
        devfn=`echo  $slot | cut -f3   -d:`
    else
        dombus=`echo $slot | cut -f1   -d:`
        devfn=`echo  $slot | cut -f2   -d:`

        if [ -d "$prefix/0000:$dombus" ]
        then
            # Try to add zero domain to the name if domain not explicitly given
            dombus="0000:$dombus"
        fi
    fi

    echo "$prefix/$dombus/$devfn"
}


save_pci()
{
    if [ $# -gt 1 ]; then
        echo "save_pci() too many arguments"
        exit 1
    fi
    check_lspci_existance
    if [ $? -ne 0 ]; then
        return 1
    fi
    local pcidev=$1 # user specified pci device to save
    local iteration_string=`lspci -m | grep Mellanox`
    if [ -n "$pcidev" ]; then
        # specific pci device was specified, remove the rest from iteration_string
        iteration_string=`echo "$iteration_string" | grep $pcidev`
        if [ ! -d $pcidir ]; then
            mkdir -p $pcidir
        fi
    else
        rm -fr $pcidir
        mkdir -p $pcidir
    fi
    echo "$iteration_string" | while read str
    do
        slot=`echo $str | cut -f1 -d' '`
        pci_file=`get_pci_file $slot`
         
        echo -n "Saving configuration for PCI device $slot"
        
        # perform mcra read of 0xf0014 in case of VSEC
        mcra pciconf-$slot 0xf0014 > /dev/null 2>&1

        # Address 0x58 contains the last cr-space address read by the conf device,
        # this address may be a semaphore, GW lock or any dangerous address,
        # so we will save the address 0xf0014 always, to protect from any unsafe read
        # or write in the save and load commands.
        setpci -s $slot 0x58.L=0xf0014 > /dev/null 2>&1

        # if the file exsist delete the file
        if [ -a $pcidir/$slot ]; then
            rm -f $pcidir/$slot
        fi
        if cp $pci_file $pcidir/$slot > /dev/null 2>&1
        then
            echo_success
        else
            echo_failure
            exit 1
        fi
    done
}

load_pci()
{
    if [ $# -gt 1 ]; then
        echo "load_pci() too many arguments"
        exit 1
    fi
    check_lspci_existance
    if [ $? -ne 0 ]; then
        return 1
    fi
    local pcidev=$1
    local iteration_string=`lspci -m | grep Mellanox`
    if [ -n "$pcidev" ]; then
        # specific pci device was specified, remove the rest from iteration_string
        iteration_string=`echo "$iteration_string" | grep $pcidev`
    fi
    echo "$iteration_string" | while read str
    do
        slot=`echo $str | cut -f1 -d' '`
        pci_file=`get_pci_file $slot`
        
        echo -n "Restoring configuration for PCI device $slot"
        if cp $pcidir/$slot $pci_file > /dev/null 2>&1
        then
            echo_success
        else
            echo_failure
            exit 1
        fi
        echo
    done
}

get_reset_addr()
{

    #### A trick to get the array that was passed to the function as an argument ####
    OLD_IFS=$IFS; IFS=''

    local array_string="$1[*]"
    local data_base_arr=(${!array_string})

    IFS=$OLD_IFS
    ###################################################################################\


    local raddr=0x0


    local element_count=${#data_base_arr[@]}
    local index=0

    while [ "$index" -lt "$element_count" ]; do
        set -- ${data_base_arr[$index]}
        devid=$1
        rst_addr=$2
        dev_expr=".*$(get_dev_name $devid)pci"

        if expr $device : $dev_expr > /dev/null 2>&1
        then
            raddr=$rst_addr
            break;
        fi

        ((index++))
    done

    echo $raddr
}

reset_pci()
{
    local orig_device=$1
    local device=$1
    local raddr=0x0

    if [ ! -e $device ]; then
        device="$mdir/$device"
        if [ ! -e $device ]; then
            echo "Device \"$orig_device\" (or \"$device\") doesn't exist"
            return 1
        fi
    fi

    raddr=$(get_reset_addr dev_id_database)
    if [ "$raddr" == "0x0" ]; then
        raddr=$(get_reset_addr live_fish_id_database)
        if [ "$raddr" == "0x0" ]; then
            echo "$device is a wrong device to reset"
            return 1
        fi
    fi

    echo -n "Reset device $device"
     
    if ${mbindir}/mcra $device $raddr 1
    then
        sleep 1
        echo_success
        echo
        return 0
    else
        echo_failure
        echo
        return 1
    fi

    return 0
}

radd()
{
    local host=$1
    local port=23108
    local proto=$2

    if [ "$proto" == "" ]; then
        proto="tcp"
    fi
    if expr $host : '.*:' > /dev/null 2>&1
    then
        set -- $(IFS=:; set -- $host; echo "$@")
        host=$1
        port=$2
    fi
    mkdir -p $mdir
    devs=`${mbindir}/mremote $host:$port L $proto`; RETVAL=$?
    for dev in $devs
    do
      if ! [[ $dev =~ [:#][0-9]+, ]] ; then  # do not add remote devices from the target host (multiple hops remote not supported)
        local fname=${mdir}/$host:$port,`echo $dev | sed -e 's/\//@/g'`
        touch $fname
      fi
    done
}

rdel()
{
    local host=$1
    local port=23108
    if expr $host : '.*:' > /dev/null 2>&1
    then
        set -- $(IFS=:; set -- $host; echo "$@")
        host=$1
        port=$2
    fi
    rm -f ${mdir}/$host:$port,*
}

cabledel()
{
    rm -f ${mdir}/*cable*
}


g_with_ib="--with_ib"
g_ibstat="ibstat"
g_ibv_devices="ibv_devices"
g_guids_list=""

is_guid_exists()
{
    guid=$1
    for g in ${g_guids_list[@]}; do
        if [ "$g" == "$guid" ]; then
            return 1
        fi
    done
    return 0
}

cableadd()
{
    WITH_IB=0
    while [ "$1" ]; do
        # Stop when we have a non flag paramater (an argument that doesn't start with -)
        if ! [[ "$1" =~ ^\- ]]; then 
            break;  
        fi
        case $1 in
            "${g_with_ib}")
                WITH_IB=1
                ;;
            *)
            echo "-E- Bad switch \"$1\" for mst ib add, please run mst help for more details."
            exit 1
        esac
        shift
    done
    hca_id=$1
    ib_port=$2
            
    cblcnt=0
    if  is_module mst_pciconf; then
        devs=/dev/mst/*pciconf*
    else
        IGNORE_FUNCS=0
        devs=$(print_ul_mdevices_info)
        ul_mode=1
    fi    
    if [ -f ${mbindir}/mlxcables ]; then
        for dev in $devs;
        do
            if ! expr match "$dev" ".*cable.*" > /dev/null 2>&1; then
                #MAX num of ports is 36 for switches
                #echo "Checking device: ${dev}"
                ports_type=`mlxcables -d ${dev} --get_dev_type`
                RC=$?
                if [[ $RC != 0 ]]; then
                    continue
                fi
                type=`echo $ports_type | cut -d' ' -f1`
                ports=`echo $ports_type | cut -d' ' -f2`
                if [[ "$ports" == "-1" ]]; then
                    ports=35
                fi
                if [[ "$type" == "HCA" ]]; then                    
                    module_num=`mlxcables -d $dev --get_module 1`
                    RC=$?
                    if [[ $RC != 0 ]]; then
                        continue
                    fi
                    cblcnt=$((  $cblcnt + 1 ))
                    cable_name=${dev}_cable_${module_num}
                    cable_dev=${cable_name}
                    if [[ "$ul_mode" == "1" ]]; then
                        cable_dev=${mdir}/${cable_name}
                    fi
                    mkdir -p ${mdir}
                    touch ${cable_dev}
                    cable_check=`${mbindir}/mlxcables -d ${cable_name} -c`
                    if [[ ${cable_check} == *FAILED* ]]; then
                        rm -f ${cable_dev}
                        cblcnt=$((  $cblcnt - 1 ))
                    fi
                elif [[ "$type" == "SW" ]]; then
                    for (( port=0; port<$ports; port++ )) do
                        cblcnt=$((  $cblcnt + 1 ))
                        cable_name=${dev}_cable_$port
                        cable_dev=${cable_name}
                        if [[ "$ul_mode" == "1" ]]; then
                            cable_dev=${mdir}/${cable_name}
                        fi
                        mkdir -p ${mdir}
                        touch ${cable_dev}
                        cable_check=`${mbindir}/mlxcables -d ${cable_name} -c`
                        if [[ ${cable_check} == *FAILED* ]]; then
                            rm -f ${cable_dev}
                            cblcnt=$((  $cblcnt - 1 ))
                        fi
                        
                    done
                fi
            fi
        done
    fi
    if [ ${WITH_IB} == "1" ]; then
        ib_devs=(${hca_id})
        ports=(${ib_port})
        if [ "${hca_id}" == "" ]; then
            if [ `is_tool_existing ${g_ibstat}` == "1" ]; then
                devs=`${g_ibstat} -l | tr '\r\n' ' '`
            elif [ `is_tool_existing ${g_ibv_devices}` == "1" ]; then
                devs=`${g_ibv_devices} | tail -n+3 | tr -d ' ' | cut  -f1 | tr '\r\n' ' '`
            else 
                echo "-E- Failed to find a tool to get the network IB interfaces"
                exit 1
            fi            
            ib_devs=("${devs}")
            ports=(1 2)
        fi
        get_ib_tools_info_index ${tool_to_use}; index=$?
        set -- ${g_tools_database[index]}
        g_tool_path=$2; g_tool_type=$3; ca_flag=$4; p_flag=$5; g_old_indexing=$6; topo_file=$7 ;
               
        
        added_nodes_type="only_mlnx"
        use_ibdr=0
        echo "-I- Discovering the fabric for connected cables ..."
        for dev in ${ib_devs[@]}; do
            for ib_port in ${ports[@]}; do
                cmd="$g_tool_path"
                hca_idx=${dev}
                if [ "${g_tool_type}" == "${G_TT_DIAGNET}" ]; then
                    # Convert from hca name (mlx4_0,mthca0,...) to index (0,1..) using ibv_devinfo.
                    if [ ${g_old_indexing} == "1" ]; then
                        get_index_for_old_diagnet ${dev}; hca_idx=$?
                    fi                   
                    cmd="$cmd -skip all"
                fi
                cmd="$cmd -${ca_flag} $hca_idx -${p_flag} $ib_port"
                rm -f ${topo_file}
                $cmd &> ${g_out_file}; RC=$?
                if [ "$RC" != "0" ]; then
                    continue
                fi
                if [ ! -f ${topo_file} ]; then
                    echo "-E- File ${topo_file} not found, Skipping ..."
                    continue
                fi
                #Run mst_ib_add script
                mkdir -p $mdir
                for d in `${mbindir}/mst_ib_add.tcl ${topo_file} ${added_nodes_type} ${g_tool_type} ${use_ibdr} $dev $ib_port --with-guids`; do
                    mdev=`echo $d | cut -d'#' -f1`
                    guid=`echo $d | cut -d'#' -f2`
                    if expr match "$mdev" ".*CA_.*" > /dev/null 2>&1
                    then
                        is_guid_exists $guid; rc=$?
                        if [ $rc == 1 ]; then
                            #echo "Ignoring $mdev, it was added before by another port"
                            continue
                        fi
                        g_guids_list+=("$guid")
                    fi
                    create_lid_cable $mdir/$mdev; c=$?
                    cblcnt=$(( $cblcnt + $c ))
                done
            done
        done
        
    fi
    echo "-I- Added $cblcnt cable devices .."
}

G_TT_DIAGNET="diagnet"
G_TT_NETDISCOVER="netdiscover"


g_ibdiagnet_tmp_path="/opt/bin/ibdiagnet"
g_out_file="/tmp/mft_discover.out"

g_ibdiag2_id="ibdiagnet2" 
g_new_ibdiag_id="new_ibdiagnet"
g_old_ibdiag_id="old_ibdiagnet"
g_ibnetdiscover="ibnetdiscover"
g_ibdiagnet_tool="ibdiagnet"
g_ibdiagnet2_lst_file="/var/tmp/ibdiagnet2/ibdiagnet2.lst"

g_tools_database=(\
    "${g_ibdiag2_id}    ${g_ibdiagnet_tmp_path} ${G_TT_DIAGNET}     i p 0 ${g_ibdiagnet2_lst_file}"
    "${g_new_ibdiag_id} ${g_ibdiagnet_tool}     ${G_TT_DIAGNET}     i p 0 ${g_ibdiagnet2_lst_file}"
    "${g_old_ibdiag_id} ${g_ibdiagnet_tool}     ${G_TT_DIAGNET}     i p 1 /tmp/ibdiagnet.lst"
    "${g_ibnetdiscover} ${g_ibnetdiscover}      ${G_TT_NETDISCOVER} C P 0 ${g_out_file}"
)

get_ib_tool_index() {
    ID=$1   
    element_count=${#g_tools_database[@]}
    index=0
    while [ "$index" -lt "$element_count" ]; do
        set -- ${g_tools_database[$index]}
        mem_id=$1
        if [ "${ID}" == "${mem_id}" ]; then
            return $index
        fi 
        ((index++))
    done
    echo "-E- Unknown discover tool \"${ID}\", to get the supported tool list run: 'mst help'"
    exit 1
}

function is_tool_existing() {
    cmd_exists=`which $1 2> /dev/null`
    if [ "$cmd_exists" == "" ]; then    
        echo "0"
    else
        echo "1"
    fi
}
function is_ibdiagnet_new() {
    ibdiagnet_tool=$1
    version="`$ibdiagnet_tool -V 2> /dev/null| head -1`"
    new_ver_regexp=\-I\-\ IBDIAGNET\ [0-9]\.[0-9]   
    if [[ "${version}" =~ ${new_ver_regexp} ]]; then
        echo "1"
    else 
        echo "0"
    fi
}

function get_ib_tools_info_index()
{
    tool_to_use=$1    
    if [ "${tool_to_use}" == "" ]; then
        if [ -f ${g_ibdiagnet_tmp_path} ] && [ `is_ibdiagnet_new ${g_ibdiagnet_tmp_path}` == "1" ]; then
            get_ib_tool_index ${g_ibdiag2_id}; return $?
        elif [ `is_ibdiagnet_new ${g_ibdiagnet_tool}` == "1" ]; then
            get_ib_tool_index ${g_new_ibdiag_id}; return $?
        elif [ `is_tool_existing ${g_ibnetdiscover}` == "1" ]; then
			get_ib_tool_index ${g_ibnetdiscover}; return $?
        elif [ `is_tool_existing ${g_ibdiagnet_tool}` == "1" ]; then
            get_ib_tool_index ${g_old_ibdiag_id}; return $?
        else 
            echo "-E- Failed to find a tool to discover the fabric (neither ${g_ibdiagnet_tool} nor ${g_ibnetdiscover} is installed on this machine)"
            exit 1
        fi
    else 
        if [ "${tool_to_use}" ==  "${g_ibdiagnet_tool}" ]; then
            if [ `is_ibdiagnet_new ${g_ibdiagnet_tool}` == "1" ]; then
                tool_to_use=${g_new_ibdiag_id}
            else 
                tool_to_use=${g_old_ibdiag_id}
            fi
        fi
        get_ib_tool_index ${tool_to_use}; return $?
    fi
 }

function get_index_for_old_diagnet() 
{
    hca_id=$1
    if [ "${hca_id}" == "" ]; then 
        return ${hca_id} 
    fi
    hca_idx=`ibv_devinfo | grep hca_id: | grep -n $hca_id | cut -f1 -d:`
    if [ "$hca_idx" == "" ]; then
        echo "-E- Failed to get hca index for hca \"$hca_id\""
        exit 1
    fi
    return ${hca_idx}
}

g_discover_tool_opt="--discover-tool"
g_topo_file_opt="--topo-file"
g_add_non_mlnx="--add-non-mlnx"
g_use_ibdr_opt="--use-ibdr"

function check_arg() {
    if [ -z $2 ] || [[ "$2" == -* ]]; then
        echo "-E- Missing parameter after \"$1\" switch."
    exit 1
    fi
}


function create_lid_cable()
{
    device=$1
    
    cblscnt=0
    if [ -f ${mbindir}/mlxcables ]; then
        #MAX num of ports is 36 for switches
        ports=`mlxcables -d ${device} --get_ports_num`
        no_port=0
        if [[ $ports < 3 ]]; then
            no_port=1
            ports=1
        fi
        
        for (( port=0; port<$ports; port++ )) do
            if [ "$no_port" == "0" ]; then
                cable_dev=${device}_cable_$port
            else
                cable_dev=${device}_cable
            fi
            touch ${cable_dev}
            cable_check=`${mbindir}/mlxcables -d ${cable_dev} -c`
                cblscnt=$(( $cblscnt + 1 ))
            if [[ ${cable_check} == *FAILED* ]]; then
                rm -f ${cable_dev}
                cblscnt=$(( $cblscnt - 1 ))
            fi
        done
    fi
    return $cblscnt
}


function ib_add()
{            
    tool_to_use=""
    topo_file=""
    added_nodes_type="only_mlnx"
    use_ibdr=0
    
    # Get the parameters
    while [ "$1" ]; do
        # Stop when we have a non flag paramater (an argument that doesn't start with -)
        if ! [[ "$1" =~ ^\- ]]; then 
            break;  
        fi
        case $1 in
            "${g_discover_tool_opt}")
                check_arg $1 $2
                tool_to_use=$2
                shift
                ;;
                
            "${g_add_non_mlnx}")
                added_nodes_type="all"
                shift
                ;;

            "${g_use_ibdr_opt}")
                use_ibdr="1"
                ;;
		             
            "${g_topo_file_opt}")
                check_arg $1 $2
                topo_file=$2
                shift
                ;;
            *)
            echo "-E- Bad switch \"$1\" for mst ib add, please run mst help for more details."
            exit 1
        esac         
        shift
    done        
    hca_id=$1
    ib_port=$2
    
    if [ "$topo_file" == "" ]; then
        get_ib_tools_info_index ${tool_to_use}; index=$?
        set -- ${g_tools_database[index]}
        g_tool_path=$2; g_tool_type=$3; ca_flag=$4; p_flag=$5; g_old_indexing=$6; topo_file=$7 ;
               
        cmd="$g_tool_path"
        hca_idx=${hca_id}
        
        if [ "${g_tool_type}" == "${G_TT_DIAGNET}" ]; then
            # Convert from hca name (mlx4_0,mthca0,...) to index (0,1..) using ibv_devinfo.        
            if [ ${g_old_indexing} == "1" ]; then
                get_index_for_old_diagnet ${hca_id}; hca_idx=$?
            fi                   
            cmd="$cmd -skip all"
	    if [ $use_ibdr == 1 ]; then
                echo "-E- Option $g_use_ibdr_opt is not supported when using ibdiagnet tool or an lst file"
		exit 1
	    fi

	else 
	    if [ $use_ibdr == 1 ]; then
                cmd="$cmd -s"
	    fi
        fi
 
        if [ "$hca_id" != "" ]; then
            cmd="$cmd -${ca_flag} $hca_idx"
            if [ "$ib_port" != "" ]; then
                cmd="$cmd -${p_flag} $ib_port"
            fi
        fi
        
        echo "-I- Discovering the fabric - Running: $cmd"
        rm -f ${topo_file}
        $cmd &> ${g_out_file}; RC=$?
        if [ "$RC" != "0" ]; then
            echo "-E- Command: \"$cmd\" failed (rc: $RC), for more details see: ${g_out_file}"
            exit 1
        fi
    else 
        if [ "${tool_to_use}" == "" ]; then 
            echo "-E- You should specify which tool you used to generate the given topofile by \"${g_discover_tool_opt} <tool>\""
            exit 1
        fi
        get_ib_tools_info_index ${tool_to_use}; index=$?
        set -- ${g_tools_database[index]}; g_tool_type=$3
    fi
 
    if [ ! -f ${topo_file} ]; then
        echo "-E- File ${topo_file} not found."
        RETVAL=1
        return
    fi
    
    ibcnt=0
    cblcnt=0
    mkdir -p $mdir
    for d in `${mbindir}/mst_ib_add.tcl ${topo_file} ${added_nodes_type} ${g_tool_type} ${use_ibdr} $hca_id $ib_port`; do
        touch $mdir/$d
        c=0
        ibcnt=$(( $ibcnt + 1 ))
    done
 
    echo "-I- Added $ibcnt in-band devices"
}

serv_start()
{
    local port=$1
    ${mbindir}/mtserver -p $port &
    mtserver_pid=$!
    ps_output=`ps -p $mtserver_pid | sed "1 d"`
    if [[ "$ps_output" == "" ]]; then
        wait $mtserver_pid
        RETVAL=$?
    fi
}

serv_stop()
{
    ps -efa | grep ${mbindir}/mtserver | grep -v grep | while read str
    do
        set -- $str
        kill $2
    done
}

check_start_args()
{
    if [[ "$1" == "" ]]; then
        return
    fi
    found=0
    read -r -a start_flags <<< "${MST_START_FLAGS} ${MST_START_HIDDEN_FLAGS}"
    for flag in "${start_flags[@]}"
    do
        if [[ "${flag}" == "[$1]" ]]; then
            found=1
        fi
    done
    if [ $found == 0 ]; then
        echo "-E- Unknown argument $1"
        exit 1
    fi
}

# See how we were called.
# --$WITH_I2CM : Create PCIe to I2C adapter as I2C master for devices that support this feature. (Was removed to hide the feature)


case "$1" in
    help)
        cat <<END

        MST (Mellanox Software Tools) service
        =====================================

   This script is used to start MST service, to stop it,
   and for some other operations with Mellanox devices
   like reset or enabling remote access.

   The mst commands are:
   -----------------------

   mst start ${MST_START_FLAGS}

       Create special files that represent Mellanox devices in
       directory ${mdir}. Load appropriate kernel modules and
       saves PCI configuration headers in directory ${pcidir}.
       After successfully completion of this command the MST driver
       is ready to work and you can invoke other Mellanox
       tools like Infiniburn or tdevmon.
       You can configure the start command by edit the configuration
       file: /etc/mft/mst.conf, for example you can rename you devices.

       Options:
       --$WITH_MSIX    : Create the msix device.
       --$WITH_UNKNOWN_ID : Do not check if the device ID is supported.
       --$WITH_DEVI2C  : Create Embedded I2C master
       --$WITH_FPGA    : Create MST device for the attached FPGA card
       
   mst stop [--$FORCE_STOP]

       Stop Mellanox MST driver service, remove all special files/directories
       and unload kernel modules.
        
        Options:
        --$FORCE_STOP : Force try to stop mst driver even if it's in use.
        
   mst restart ${MST_START_FLAGS}

       Just like "mst stop" followed by  "mst start ${MST_START_FLAGS}"

   mst server start [port]
       Start MST server to allow incoming connection.
       Default port is 23108

   mst server stop
       Stop MST server.

   mst remote add <hostname>[:port]
       Establish connection with specified host on specified port
       (default port is 23108). Add devices on remote peer to local
       devices list. <hostname> may be host name as well as an IP address.

   mst remote del <hostname>[:port]
       Remove all remote devices on specified hostname. <hostname>[:port] should
       be specified exactly as in the "mst remote add" command.

   mst ib add [OPTIONS] [local_hca_id] [local_hca_port] 
       Add devices found in the IB fabric for inband access.
       Requires OFED installation and an active IB link.
           If local_hca_id and local_hca_port are given, the IB subnet connected
           to the given port is scanned. Otherwise, the default subnet is scanned.
       OPTIONS:
            ${g_discover_tool_opt} <discover-tool>: The tool that is used to discover the fabric.
                                             Supported tools: $g_ibnetdiscover, $g_ibdiagnet_tool. default: ${g_ibdiagnet_tool}
            ${g_add_non_mlnx} : Add non Mellanox nodes.
            ${g_topo_file_opt} <topology-file>: A prepared topology file which describes the fabric.
                         For $g_ibnetdiscover: provide an output of the tool.
                         For $g_ibdiagnet_tool: provide LST file that $g_ibdiagnet_tool generates.
            ${g_use_ibdr_opt}  : Access by direct route MADs. Available only when using ibnetdiscover tool, for 5th generation devices.
            NOTE: if a topology file is specified, device are taken from it.
                  Otherwise, a discover tool is run to discover the fabric.

   mst ib del
       Remove all inband devices.
   
   mst cable add [OPTIONS] [params]
       Add the cable that are connected to the devices, which support the MCIA access register.
       There are an option to add the cables found in the IB fabric for Cable Info access,
       requires OFED installation and active IB links.
            If local_hca_id and local_hca_port are given, the IB subnet connected
            to the given port is scanned. Otherwise, all the devices will be scanned.
        OPTIONS:
            ${g_with_ib}: Add the inband cables in addition to the local PCI devices.
                params: [local_hca_id] [local_hca_port]
       
   mst cable del
       Remove all cable devices.
   
   mst status

       Print current status of Mellanox devices

       Options:
       -v run with high verbosity level (print more info on each device)

   mst save

       Save PCI configuration headers in directory ${pcidir}.

   mst load

        Load PCI configuration headers from directory ${pcidir}.

   mst version 

       Print the version info

END
        ;;
    save)
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        if [ $# -eq 2 ]
        then
        save_pci $2; RC=$?
        else
        save_pci; RC=$?
        fi
        if [ "$RC" != "0" ]; then
           RETVAL=1
        fi
        ;;
    load)
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        if [ $# -eq 2 ]
        then
        load_pci $2; RC=$?
        else
        load_pci; RC=$?
        fi
        if [ "$RC" != "0" ]; then
           RETVAL=1
        fi
        ;;
    reset)
        # We are expecting for device name
        if [ $# -lt 2 ]; then
            echo "Please specify device name"
            devcnt=0
            echo
            echo "Available devices are:"
            echo "----------------------"
            echo
            for dev in ${mdir}/*
            do
                if [ -r "$dev" ]; then
                    if ! expr "$dev" : '.*\(vtop\|ddr\|uar\)' > /dev/null 2>&1
                    then
                        cat $dev | head -n 1
                        devcnt=$((  $devcnt + 1 ))
                    fi
                fi
            done
            if [ ${devcnt} -eq 0 ]; then
                echo "    Sorry, no available devices found."
            fi
            exit 1
        fi

        # Check that we are root
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        if save_pci 
        then
            if reset_pci $2
            then
                load_pci
            else
                echo "-E- Failed to reset the pci device $2"
                exit 1
            fi
        else 
            echo "-E- Failed to save the pci configuration headres before resetting $2."
            RETVAL=1
        fi
        ;;
    server)
        # We are expecting subsommand here
        if [ $# -lt 2 ]; then
            echo "Please specify subcommand (start or stop)"
            exit 1
        fi
        case "$2" in
            start)
                cat /proc/devices | grep 'mst' > /dev/null
                if [ $? -ne 0 ]; then
                    mst start || exit 1
                fi

                if [ $# -lt 3 ]; then
                    serv_start 23108
                else
                    serv_start $3
                fi
            ;;
            stop)
                if [ $# -gt 2 ]; then
                    echo "Unknown option/s: ${@:3}"
                    exit 1
                fi
                serv_stop
            ;;
            *)
                echo "Subcommand should be start or stop"
                RETVAL=1
            ;;
        esac
        ;;
    remote)
        # We are expecting command and host name
        if [ `id -u` -ne 0 ]; then 
            echo "-E- You must be root to add/remove remote devices"
            exit 1   
        fi

        if [ $# -lt 3 ]; then
            echo "Please specify subcommand (add or del) and remote host name or its IP address"
            exit 1
        fi
        
        case "$2" in
            add)
                radd $3 $4
            ;;
            del)
                rdel $3
            ;;
            *)
                echo "Subcommand should be add or del"
                RETVAL=1
            ;;
        esac
        ;;
    ib)
        # We are expecting subsommand here
        if [ `id -u` -ne 0 ]; then 
            echo "-E- You must be root to add/remove ib devices"
            exit 1   
        fi
        if [ $# -lt 2 ]; then
            echo "Please specify subcommand (add or del)"
            exit 1
        fi
        case "$2" in
            add)
                shift 2
                ib_add $@
            ;;
            del)
                #rm -f $mdir/SW_* $mdir/CA_*
                ls $mdir/SW_* $mdir/CA_* 2> /dev/null | grep -v cable | xargs rm -f 
            ;;
            *)
                echo "Subcommand should be add or del"
                RETVAL=1
            ;;
        esac
        ;;
    cable)
        # We are expecting command and host name
        if [ `id -u` -ne 0 ]; then 
            echo "-E- You must be root to add/remove cable devices"
            exit 1   
        fi

        if [ $# -lt 2 ]; then
            echo "Please specify subcommand (add or del)"
            exit 1
        fi
        
        case "$2" in
            add)
                shift 2
                cableadd $@
            ;;
            del)
                cabledel $3
            ;;
            *)
                echo "Subcommand should be add or del"
                RETVAL=1
            ;;
        esac
        ;;
    start)
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        check_start_args $2
        start $2 $3
        ;;
    stop)
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        stop $2
        rm -fr $pcidir
        ;;
    status)
            print_status $2
        ;;
    version)
    	if [ -f $BASH_VERSION_LIB_PATH ]; then
            source $BASH_VERSION_LIB_PATH
            print_version_string mst
        else
            echo "N/A"
        fi    	   
        ;; 
    restart)
        if [ `id -u` -ne 0 ]; then
            echo "You must be root to do that"
            exit 1
        fi
        check_start_args $2
        stop
        start $2 $3
        ;;
    *)
        echo "Usage:"
        echo "    $0 {start|stop|status|remote|server|restart|save|load|help|version}"
        echo
        echo "Type \"$0 help\" for detailed help"
        RETVAL=1
esac
exit $RETVAL
