#!/bin/sh
# $Id: collect_data_kvm.sh
#
# data collection script for SPECvirt_sc2013 under KVM
#
# TODO:
# handle physical client with no vclient with a graceful way
#
# set -x
OUTPUT_DIR=/tmp/data-collection-$(date +"%Y%m%d-%H%M%S")
CONFIG=/opt/SPECvirt/collect_data.cfg
#DC_SCRIPT="/opt/SPECvirt/ddcollect_data_kvm.sh"
DC_SCRIPT=$0
#VM_EXT=dl385gen10

# temp settings
SCRIPT_REMOTE=/tmp/collect_data_kvm.sh
BASEDIR_SYS=/tmp/data-collection-sys
BASEDIR_SW=/tmp/data-collection-sw

if echo "$0" | grep -q '^\/.*'
then
    SCRIPT_LOCAL="$0"
else
    SCRIPT_LOCAL="$(pwd)/$0"
fi

sanity_check()
{
    nmcli connection up eth0
    data_collect_log "Checking hostname..." 3
    ping_host "sut"

    for client in $(gen_list $PHYCLIENT_LIST)
    do
        ping_host "phyclient${client}"
    done

    for tile in $(seq $TILE_LIST)
    do
        ping_host "client${tile}" 
        ping_host "bclient${tile}" 

        vmlist="appserver batchserver infraserver mailserver webserver"
        rest=$[$tile%4]
        if [ "$rest" -eq 1 ]; then
            vmlist="$vmlist dbserver"
        fi

#        for host in $vmlist
#        do
#            ping_host "${host}${tile}"
#        done
     done

    return 0;
}

ping_host()
{
    local host="$1"
    echo ${host}
    if ! ping -w2 -c1 ${host} &>/dev/null
    then
        data_collect_log "${host} not pingable. Please check /etc/hosts!" 2
        exit 2
    fi
}

#
# not used yet
#
setup_host_key() 
{   
#    local CLIENT="$1"
#    local PASS="$2"

#    expect <<ENDEXP
#spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@${CLIENT}
#expect "*?assword:*"
#send -- "${PASS}\r"
#send -- "\r"
expect eof   
ENDEXP
}

#
# this will setup the key auth
#
prepare_run_env()
{
    setup_host_key sut

    for client in $(gen_list $PHYCLIENT_LIST)
    do
        setup_host_key "phyclient${client}"
    done

    for tile in $(seq $TILE_LIST)
    do
        setup_host_key "client${tile}" 

        vmlist="appserver batchserver infraserver mailserver webserver"
        rest=$[$tile%4]
        if [ "$rest" -eq 1 ]; then
            vmlist="$vmlist dbserver"
        fi

        for host in $vmlist
        do
            setup_host_key "${host}${tile}"
        done
    done
echo "hi"
}

build_dir_tree()
{
    while read DIR
    do
        mkdir -p $DIR
    done <<ENDLINE
./VIRT
./PHYS
./PHYS/SUT
./PHYS/SUT/vm-xml
./PHYS/Clients
./Collection_scripts
ENDLINE
}

data_collect_log()
{
    local text="$1"
    local class="$2"
    shift 2

    if [ "$class" = "2" ]
    then
        echo "$text" 1>&2
    elif [ "$class" = "3" ]
    then
        echo $* "$text"
    fi

    echo "$(date "+%b %d %Y %T") $text" >> $OUTPUT_DIR/data-collection.log
}

gen_list()
{
    local IFS=","
    local LIST="$1"

    for i in $LIST
    do
        if echo $i | grep -q '^[0-9]*-[0-9]*$'
        then
            seq ${i%-*} ${i#*-}
        else
            echo $i
        fi
    done
}

collect_proc_info()
{
    local procfile="$1"
    if [ ! -e "$procfile" ]
    then
        return 1
    fi

    local DIR=$(dirname $procfile)
    local FILE=$(basename $procfile)

    DIR=${DIR#/}
    mkdir -p "$DIR" &>/dev/null

    if [ $? -ne 0 ]
    then
        return 1
    fi

    cat $procfile > "$DIR/$FILE"
}

collect_sys_info_sut()
{
    mkdir OS SOFTW Storage Network
# hw and device info
    lspci > OS/lspci.txt

# os info
#    hostname > hostname.txt

# network
    ifconfig > Network/ifconfig.txt
    route -n > Network/route.txt
    #brctl show > Network/brctl.txt 
    bridge link show > Network/bridge.txt  

# storage and filesystem
    collect_proc_info "/proc/partitions"
    vgdisplay -v > Storage/vgdisplay.txt 2>/dev/null
    lvdisplay > Storage/lvdisplay.txt 2>/dev/null
    mount > Storage/mount.txt
    df > Storage/df.txt
    lsblk > Storage/lsblk.txt

# packages and service
    rpm -qa > OS/rpmpkgs.txt
    #chkconfig --list > chkconfig.txt
    sysctl -a > OS/sysctl.txt
    systemctl list-unit-files > OS/systemctl-services.txt

# runtime info
    ps -ef | grep qemu > SOFTW/running-vms.txt
    virsh list > SOFTW/virsh-list.txt

# proc info
    collect_proc_info "/proc/cpuinfo"
    collect_proc_info "/proc/devices"
    collect_proc_info "/proc/meminfo"
    collect_proc_info "/proc/modules"
    collect_proc_info "/proc/swaps"
    collect_proc_info "/proc/version"
    collect_proc_info "/proc/cmdline"

# other static configuration and logs 
    find /etc/fstab /etc/os-release /etc/iproute2 /etc/hosts /etc/inittab /etc/sysctl.conf /etc/nsswitch.conf /etc/security/limits.conf /etc/rc.d/init.d /etc/rc.d/rc3.d /etc/bashrc /etc/exports /etc/rc.d/rc.local /etc/tuned/active_profile /boot/efi/EFI/bclinux/grub.cfg /boot/efi/EFI/bclinux/grubenv 2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
mv etc proc boot OS/

# clear zero size file
find . -size 0 -exec rm -f {} \;

}

collect_sys_info_phylient()
{
    mkdir OS SOFTW Storage Network
# hw and device info
    lspci > OS/lspci.txt

# os info
#    hostname > hostname.txt

# network
    ifconfig > Network/ifconfig.txt
    route -n > Network/route.txt
   #brctl show > Network/brctl.txt 
    bridge link show > Network/bridge.txt

# storage and filesystem
    collect_proc_info "/proc/partitions"
    vgdisplay -v > Storage/vgdisplay.txt 2>/dev/null
    lvdisplay > Storage/lvdisplay.txt 2>/dev/null
    mount > Storage/mount.txt
    df > Storage/df.txt

# packages and service
    rpm -qa > OS/rpmpkgs.txt
    #chkconfig --list > chkconfig.txt
    sysctl -a > OS/sysctl.txt
    systemctl list-unit-files > OS/systemctl.txt

# runtime info
    ps -ef | grep qemu > SOFTW/running-vms.txt
    virsh list > SOFTW/virsh-list.txt

# proc info
    collect_proc_info "/proc/cpuinfo"
    collect_proc_info "/proc/devices"
    collect_proc_info "/proc/meminfo"
    collect_proc_info "/proc/modules"
    collect_proc_info "/proc/swaps"
    collect_proc_info "/proc/version"
    collect_proc_info "/proc/cmdline"

# other static configuration and logs   
    find /etc/fstab /etc/os-release /etc/iproute2 /etc/hosts /etc/inittab /etc/sysctl.conf /etc/nsswitch.conf /etc/security/limits.conf /etc/rc.d/init.d /etc/rc.d/rc3.d /etc/bashrc /etc/exports /etc/rc.d/rc.local /boot/efi/EFI/bclinux/grub.cfg /boot/efi/EFI/bclinux/grubenv 2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
mv etc proc boot OS/

# clear zero size file

find . -size 0 -exec rm -f {} \;

}

collect_sys_info_vm()
{   
# hw and device info
    lspci > lspci.txt

# os info
#    hostname > hostname.txt

# network
    ifconfig > ifconfig.txt
    route -n > route.txt
#   brctl show > brctl.txt 
    bridge link show > bridge.txt

# storage and filesystem
    collect_proc_info "/proc/partitions"
    mount > mount.txt
    df > df.txt

# packages and service
    rpm -qa > rpmpkgs.txt
    # chkconfig --list > chkconfig.txt

# runtime info
    #ps -ef | grep qemu > running-vms.txt
    #virsh list > virsh-list.txt
    #tuned-adm active > tuned-adm.txt
# proc info
    collect_proc_info "/proc/cpuinfo"
    collect_proc_info "/proc/meminfo"
    collect_proc_info "/proc/modules"
    collect_proc_info "/proc/swaps"
    collect_proc_info "/proc/version"
    collect_proc_info "/proc/cmdline"

# other static configuration and logs 
    find /etc/fstab /etc/os-release /etc/iproute2 /etc/hosts /etc/inittab /etc/sysctl.conf /etc/nsswitch.conf /etc/security/limits.conf /etc/rc.d/init.d /etc/rc.d/rc3.d /etc/bashrc	/etc/exports /etc/rc.d/rc.local /etc/tuned/active_profile /boot/efi/EFI/redhat/grub.cfg 2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null

# clear zero size file
find . -size 0 -exec rm -f {} \;

}

collect_client_soft_info_vm()
{
mkdir SPECbatch SPECimap SPECjAppServer2004 SPECweb2005
cp -pv /opt/SPECbatch/Test.config $BASEDIR_SW/SPECbatch
cp -pv /opt/SPECimap/IMAP_config.rc $BASEDIR_SW/SPECimap
cp -pv /opt/SPECjAppServer2004/config/run.properties $BASEDIR_SW/SPECjAppServer2004
cp -pv /opt/SPECjAppServer2004/config/default.env $BASEDIR_SW/SPECjAppServer2004
cp -pv /opt/SPECweb2005/SPECweb_Support.config $BASEDIR_SW/SPECweb2005
cp -pv /opt/SPECweb2005/Test.config $BASEDIR_SW/SPECweb2005;
}

collect_remote_client_software_config()
{
    local host="$1"

    scp -pq $SCRIPT_LOCAL $host:$SCRIPT_REMOTE   
    ssh $host sh $SCRIPT_REMOTE clientsoftware
    ssh $host tar -cz /tmp/data-collection-sw 2>/dev/null | tar -xz 2>/dev/null
    mv tmp/data-collection-sw/* .
    rm -fr tmp
}

collect_remote_vm_sys_info()
{
    local host="$1"
    if [[ "$host" == *client* ]]; then  
    scp -pq $SCRIPT_LOCAL $host:$SCRIPT_REMOTE
    ssh $host sh $SCRIPT_REMOTE sysinfo_vm
    ssh $host tar -cz /tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null
    elif [ "$type" == "web" ] || [ "$type" == "infra" ]; then
    ssh bclient$2 "scp -pq $SCRIPT_REMOTE $host:$SCRIPT_REMOTE"   
    ssh bclient$2 "ssh $host sh $SCRIPT_REMOTE sysinfo_vm"    
    ssh bclient$2 "ssh $host tar -cz /tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null"
    ssh bclient$2 tar -cz tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null  
    ssh bclient$2 "ssh $host rm -fr /tmp/data-collection-sys"
    ssh bclient$2  rm -fr tmp/data-collection-sys
    else
    ssh client$2 "scp -pq $SCRIPT_REMOTE $host:$SCRIPT_REMOTE"   
    ssh client$2 "ssh $host sh $SCRIPT_REMOTE sysinfo_vm"   
    ssh client$2 "ssh $host tar -cz /tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null"
    ssh client$2 tar -cz tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null  
    ssh client$2 "ssh $host rm -fr /tmp/data-collection-sys"
    ssh client$2  rm -fr tmp/data-collection-sys
    fi
    mv tmp/data-collection-sys/* .
    rm -fr tmp
}

collect_remote_sys_info()
{
    local host="$1"   
    scp -pq $SCRIPT_LOCAL $host:$SCRIPT_REMOTE   
    ssh $host sh $SCRIPT_REMOTE sysinfo_sut    
    ssh $host tar -cz /tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null   
    mv tmp/data-collection-sys/* .     
    rm -fr tmp
}

collect_remote_sys_info_phyclient()
{
    local host="$1"

    scp -pq $SCRIPT_LOCAL $host:$SCRIPT_REMOTE
    ssh $host sh $SCRIPT_REMOTE sysinfo_phyclient
    ssh $host tar -cz /tmp/data-collection-sys 2>/dev/null | tar -xz 2>/dev/null
    mv tmp/data-collection-sys/* .
    rm -fr tmp
}

collect_SUT()
{
    cd SUT
    data_collect_log "Collecting SUT info..." 3
    data_collect_log "Entering $(pwd)..."

    collect_remote_sys_info "sut"

    cd ..
}

collect_clients()    
{   
    cd Clients
    data_collect_log "Collecting physical clients info..." 3
    data_collect_log "Entering $(pwd)..."

    for client in $(gen_list $PHYCLIENT_LIST)
    do
        dir="Client${client}"
        mkdir $dir && cd $dir
        data_collect_log "Collecting $dir..." 
        collect_remote_sys_info_phyclient "phyclient${client}" &  
       #cd ..
     #done

    wait
    data_collect_log "Collecting virtual clients info..." 3
                    if [ ${client} = "1" ]; then
                    local CLIENT=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 33 34 35 36)
                    elif [ ${client} = "2" ]; then
                      CLIENT=(17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 37 38 39 40)
                    fi

    for vclient in ${CLIENT[@]}
        do
            dir="Tile${vclient}"
            mkdir $dir && cd $dir
            
            dir1="client${vclient}"   
            mkdir $dir1 && cd $dir1
            mkdir OS && cd OS 
            data_collect_log "Collecting client $dir..."
            collect_remote_vm_sys_info "client${vclient}" &
            cd ../
            collect_remote_client_software_config "client${vclient}" &
            cd ../
            
            dir1="bclient${vclient}"
            mkdir $dir1 && cd $dir1
            mkdir OS && cd OS
            data_collect_log "Collecting bclient $dir..."
            collect_remote_vm_sys_info "bclient${vclient}" &
            cd ../
            collect_remote_client_software_config "bclient${vclient}" &
            cd ../
            cd ../ 
        done
        cd ../   
    done
    cd ../
}

collect_physical_configuration()
{
    cd PHYS
    data_collect_log "Entering $(pwd)..."

    collect_SUT
    collect_clients

    cd ..
}

collect_local_software_config()
{
    local type="$1"

    case $type in

        "app")
	    mkdir {glassfish,tomcat,SPECjAppServer2004,SPECjAppServer2004/config}
            java -version 2> app-javaVersion.out

           curl -s --connect-timeout 10 -m 20  http://localhost:8080 | grep 'Apache Tomcat' > tomcat/TomcatVersion.out
           /opt/glassfish4/bin/asadmin --passwordfile /opt/SPECjAppServer2004/bin/passwordfile version > glassfish/GlassFishVersion.out

           pushd /opt/glassfish4/glassfish/domains/
           tar cf -	\
spec2004-1/{config,autodeploy}  \
2> /dev/null | tar xf - -C $BASEDIR_SW/glassfish &>/dev/null

	    pushd $BASEDIR_SW/glassfish/spec2004-1
	    find ./ -name "*SPECjAppServer_ItemBrowserSes_*" | xargs rm -rf - 
	    find ./ -name \"*.dat*\" | xargs rm -rf -
	    rm -f config/*com_sun_appserv_pid
	    rm -f config/.*com_sun_appserv_pid

           pushd /opt/tomcat
           tar cf -	\
{conf,webapps/Emulator.war}	\
2> /dev/null | tar xf - -C $BASEDIR_SW/tomcat &>/dev/null

	    cp -v /opt/SPECjAppServer2004/config/{run.properties,glassfish.env} $BASEDIR_SW/SPECjAppServer2004/config
	    # cp -pv /opt/SPECjAppServer2004/config/{*.properties,default.env} $BASEDIR_SW/SPECjAppServer2004/config
        ;;

        "db")
# mkdir schema
# cp -pv /home/db2inst1/sqllib/db2nodes.cfg .
# cp -pv /home/db2inst1/sqllib/profile.env .
# cp -pv /home/db2inst1/tun.sh .
# cp -pv /home/db2inst1/db2.dbm.cfg .
# cp -pv /home/db2inst1/db2.db.cfg .
# cp -prv /opt/SPECjAppServer2004/schema/db2  $BASEDIR_SW/schema

             java -version 2> db-javaVersion.out

           psql --version > psql-Version.txt
        #    cp -p /var/lib/pgsql/9.2/data/postgresql.conf .
           cp -p /var/lib/pgsql/data/postgresql.conf .
	    pushd /opt/SPECjAppServer2004/
	
           tar cf -	\
schema/postgresql bin/glassfish  \
2> /dev/null | tar xf - -C $BASEDIR_SW &>/dev/null
        ;;

        "batch")
            java -version 2> batch-javaVersion.out
            java -jar /opt/SPECpoll/pollme.jar -v > SPECpollVersion.out

	    mkdir -p SPECbatch/cpu2006
	    cp -pv /opt/SPECbatch/cpu2006-virt/{virt_sc2013_run.sh,config/cpu2006-virt.cfg} SPECbatch/cpu2006/
	    cp -pv /opt/SPECbatch/Test.config SPECbatch/cpu2006/

            #touch emptyfile

            find 	\
2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
        ;;

        "infra")
            java -version 2> infra-javaVersion.out
           # sum /var/www/fcgi-bin/besim_fcgi.fcgi > sum.besim_fcgi
           # ldd /var/www/fcgi-bin/besim_fcgi.fcgi > ldd.besim_fcgi

            find /etc/httpd/conf \
/etc/httpd/conf.d 	\
2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
            httpd -V > etc/httpd/httpdVersion.out

        ;;

        "mail")
            java -version 2> mail-javaVersion.out

            find  \
/etc/dovecot \
/etc/init.d/dovecot 	\
2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
            dovecot --version > etc/dovecotVersion.out
            dovecot --build-options >> etc/dovecotVersion.out
			

	    mkdir SPECimap
	    cp -pv /opt/SPECimap/IMAP_config.rc SPECimap/
	    cp -pv /opt/SPECimap/IMAP_fixed.rc SPECimap/

        ;;

        "web")
####apache code
            php --info > phpinfo.out 2>/dev/null
            # grep -i "^[ ]*DocumentRoot" /etc/httpd/conf/httpd.conf |tr -d '\"' |xargs lsattr -R - 2>/dev/null  |grep support |grep -v dir000 > lsattr.out

#            find /etc/httpd/conf \
#/etc/httpd/conf.d \
		cp -pv /etc/php.ini .
/var/www/html/support/init_vars.php	\
2>/dev/null | cpio -o 2>/dev/null | cpio -idm --no-absolute-filenames &>/dev/null
#            httpd -V > etc/httpd/httpdVersion.out
	    cp -pv /var/www/html/support/init_vars.php .

	    mkdir SPECweb2005
	    cp -pv /opt/SPECweb2005/Test.config SPECweb2005/
	    cp -pv /opt/SPECweb2005/SPECweb_Support.config SPECweb2005/

            mkdir nginx
            mkdir nginx/conf.d
	    nginx -V 2>&1 |tee nginx/nginxVersion.out
            cp -pv /etc/nginx/nginx.conf nginx/
	    cp -pv /etc/nginx/conf.d/default.conf nginx/conf.d/
	    cp -pv /etc/nginx/conf.d/ssl.conf nginx/conf.d/
            mkdir php-fpm
            mkdir php-fpm/php-fpm.d
	    php-fpm -v > php-fpm/php-fpmVersion.out
	    cp -pv /usr/local/etc/php-fpm.conf php-fpm/
	    cp -pv /usr/local/etc/php-fpm.d/www.conf php-fpm/php-fpm.d/	
          
            java -version 2> web-javaVersion.out   
     
         ;;

    esac
}

collect_remote_software_config()
{
    local host="$1"
    local type="$2"
    local tile="$3"
   ssh client${tile} "scp -q $SCRIPT_REMOTE $host:$SCRIPT_REMOTE"  

if [ "$type" == "web" ] || [ "$type" == "infra" ]; then
    ssh bclient${tile} "ssh $host sh $SCRIPT_REMOTE "software" $type"   
    ssh bclient${tile} "ssh $host tar -cz /tmp/data-collection-sw 2>/dev/null | tar -xz 2>/dev/null"
    ssh bclient${tile} tar -cz tmp/data-collection-sw 2>/dev/null | tar -xz 2>/dev/null
    ssh bclient${tile} "ssh $host rm -fr /tmp/data-collection-sw"
    ssh bclient${tile}  rm -fr tmp/data-collection-sw
else
    ssh client${tile} "ssh $host sh $SCRIPT_REMOTE "software" $type"
    ssh client${tile} "ssh $host tar -cz /tmp/data-collection-sw 2>/dev/null | tar -xz 2>/dev/null" 
    ssh client${tile} tar -cz tmp/data-collection-sw 2>/dev/null | tar -xz 2>/dev/null
    ssh client${tile} "ssh $host rm -fr /tmp/data-collection-sw"
    ssh client${tile}  rm -fr tmp/data-collection-sw
fi
    mv tmp/data-collection-sw/* .
    rm -fr tmp
}

collect_software_config()
{
    local type="$1"
    local tile="$2"
    if [ "$type" = "db" ] && [ "$tile" -gt 2 ]; then
    result=$(echo "scale=2; $((tile+1)) / 4 + 0.5" | bc | awk '{printf "%d\n", int($1 + 0.5)}')
    local host="${type}server${result}" 
    else
    local host="${type}server${tile}"   
    fi
    data_collect_log "Collecting software configuration on ${host}..."
    mkdir SOFTW && cd SOFTW
    collect_remote_software_config "$host" "$type" "$tile" &
    cd ..
}

collect_vm_config()
{
    local type="$1"
    local tile="$2"
    if [ "$type" = "db" ] && [ "$tile" -gt 2 ]; then
    result=$(echo "scale=2; $((tile+1)) / 4 + 0.5" | bc | awk '{printf "%d\n", int($1 + 0.5)}')
    local host="${type}server${result}" 
    else
    local host="${type}server${tile}"   
    fi
    data_collect_log "Collecting vm configuration on ${host}..."
    mkdir OS && cd OS
    collect_remote_vm_sys_info "$host" "$tile" &
    cd ..
}


collect_one_tile()    
{   
    local tile="$1"
    local host=""
    data_collect_log "tile number: ${tile}"
    vmlist="app batch infra mail web"
    rest=$[$tile%4]
    if [ "$rest" -eq 1 ]; then
        vmlist="$vmlist db"
    fi

    for host in $vmlist
    do
        mkdir $host && cd $host
        #collect KVM xml for tile
        if [ ${tile} -lt 10 ]
	then
	  
	    if [ "$rest" -eq 1 ]; then
	     for i in app batch infra mail web db; do ssh sut "virsh dumpxml ${i}server${tile} > /tmp/${i}0${tile}.xml";done
	     scp -p sut:/tmp/*0${tile}.xml ../../../PHYS/SUT/vm-xml/
	     for i in  app batch infra mail web db; do ssh sut "rm -rf /tmp/${i}0${tile}.xml";done 
	   else
	     for i in  app batch infra mail web; do ssh sut "virsh dumpxml ${i}server${tile} > /tmp/${i}0${tile}.xml";done
             scp -p sut:/tmp/*0${tile}.xml ../../../PHYS/SUT/vm-xml/
             for i in  app batch infra mail web; do ssh sut "rm -rf /tmp/${i}0${tile}.xml";done
	   fi 
	else
	     if [ "$rest" -eq 1 ]; then
	     for i in app batch infra mail web db; do ssh sut "virsh dumpxml ${i}server${tile} > /tmp/${i}${tile}.xml";done
	     scp -p sut:/tmp/*${tile}.xml ../../../PHYS/SUT/vm-xml/
	     ssh sut "rm -f /tmp/${host}0${tile}.xml"
	     for i in app batch infra mail web db; do ssh sut "rm -rf /tmp/${i}${tile}.xml";done 
	   else
	     for i in app batch infra mail web; do ssh sut "virsh dumpxml ${i}server${tile} > /tmp/${i}${tile}.xml";done
             scp -p sut:/tmp/*${tile}.xml ../../../PHYS/SUT/vm-xml/
	     ssh sut "rm -f /tmp/${host}0${tile}.xml"
             for i in app batch infra mail web; do ssh sut "rm -rf /tmp/${i}${tile}.xml";done
	   fi 
	fi

        data_collect_log "Collecting ${host} in tile${tile}..."

        collect_software_config "$host" "$tile"    
        collect_vm_config "$host" "$tile"   

        cd ..
    done

    # per tile wait
    #wait
}

collect_virtual_configuration()
{
	cd VIRT
    data_collect_log "Entering $(pwd)..."

    for tile in $(seq $TILE_LIST)
    do
        mkdir "Tile${tile}" && cd "Tile${tile}"
        data_collect_log "Collecting tile${tile}..." 3
        data_collect_log "Entering $(pwd)..."
        collect_one_tile "${tile}"
        cd ..
    done

    cd ..
}

collect_support_files()
{
	cd Collection_scripts
	cp $DC_SCRIPT ./
#	scp -r appserver1:/opt/SPECjAppServer2004/bin/glassfish ./
cp /opt/SPECvirt/collect_data_kvm.sh ./
cp /opt/SPECvirt/collect_data.cfg ./

	cd ..
}


#################################################################
# main routine starts here
#

if [ "$1" = "sysinfo" ]
then
    rm -fr $BASEDIR_SYS 2>/dev/null && mkdir $BASEDIR_SYS 2>/dev/null && cd $BASEDIR_SYS
    collect_sys_info   
elif [ "$1" = "sysinfo_sut" ]
then
    rm -fr $BASEDIR_SYS 2>/dev/null && mkdir $BASEDIR_SYS 2>/dev/null && cd $BASEDIR_SYS
    collect_sys_info_sut   
elif [ "$1" = "sysinfo_phyclient" ]
then
    rm -fr $BASEDIR_SYS 2>/dev/null && mkdir $BASEDIR_SYS 2>/dev/null && cd $BASEDIR_SYS
    collect_sys_info_phylient   
elif [ "$1" = "clientsoftware" ]
then
    rm -fr $BASEDIR_SW 2>/dev/null && mkdir $BASEDIR_SW 2>/dev/null && cd $BASEDIR_SW
    collect_client_soft_info_vm   
elif [ "$1" = "sysinfo_vm" ]
then
    rm -fr $BASEDIR_SYS 2>/dev/null && mkdir $BASEDIR_SYS 2>/dev/null && cd $BASEDIR_SYS
    collect_sys_info_vm   
elif [ "$1" = "software" ]
then
    rm -fr $BASEDIR_SW 2>/dev/null && mkdir $BASEDIR_SW 2>/dev/null && cd $BASEDIR_SW
    collect_local_software_config "$2"  
elif [ "$1" = "prepare" ]
then
    prepare_run_env
else

    if [ -r "$CONFIG" ]
    then
        source $CONFIG
    else
        data_collect_log "Config $CONFIG not readable, please check!" 2
        exit 1
    fi

    rm -fr $OUTPUT_DIR 2>/dev/null && mkdir $OUTPUT_DIR 2>/dev/null && cd $OUTPUT_DIR
    sanity_check

    build_dir_tree    

    date > Data-Collection-Start.txt

    collect_support_files   

    collect_physical_configuration  

    collect_virtual_configuration  

    # final wait
    wait

    # Remove the obsolete files
    pwd
    cd $OUTPUT_DIR
    find ./ -name "*.bak*" | xargs rm -rfv -
    find ./ -name "*.orig*" | xargs rm -rfv -
    find ./ -name "*_sc2010*" | xargs rm -rfv -
    find ./ -name "*.rpmsave*" | xargs rm -rfv -
    find ./ -name "*.rpmnew*" | xargs rm -rfv -
    find ./ -name "*before*" | xargs rm -rfv -

    date > Data-Collection-End.txt

    data_collect_log "Data has been collected under $OUTPUT_DIR" 3

    mv data-collection.log /tmp/

    exit 0
fi

