Quickly check hundreds of Linux VM's for read-only filesystems

Sometimes a NAS cluster failover event for routine maintenance may take too long; or you bought the wrong storage solution; or you have a major network or SAN fault... or something else.

Any of these situations can cause your virtual machines to remount their filesystems "read-only".  The proper thing (apart from fixing your infrastructure) to make your VM's more tolerant is to configure SCSI timeouts within each VM, in the case of VMware, or in the case of KVM or Xen on NFS, to change the NFS mount options. (see http://discussions.citrix.com/topic/344713-linux-vm-storage-going-read-only-on-nfs-shares-and-a-proposed-fix-in-xenserver/ for the Citrix XenServer hack.)

To detect this "read-only filesystem" rapidly, I wrote a little script I call "ckrofs".  You use it as follows:

ckrofs ...
or
ckrofs `cat hostlist`



Here's the code:

#!/bin/bash

HostList=`echo ${@} | tr \\\n " "`

PARALLELISM=20

function remoteCheck()
{ 
   TargetHost=$1
   ThisHostOutput="### $TargetHost ###"
   ThisHostOutput+="$(ssh -q $TargetHost 'for targetfs in `mount | egrep "ext" | cut -d " " -f 3`; do touch $targetfs/foofile > /dev/null 2>&1 && rm -f $targetfs/foofile  || echo -n " $targetfs appears to be readonly #";  done ' 2>&1 || echo ' unable to log in to host' )"
   echo $ThisHostOutput
}

export -f remoteCheck


echo ${HostList} | xargs -P $PARALLELISM -d" " --replace="HOST" /bin/bash -c 'remoteCheck HOST'

Get SSL Certificate Vitals in Linux

This script will let you programmatically get a certificate start date, number of days remaining, and certificate hash, suitable for example for automated checking for expired or changed certificates, as with Zabbix:

#!/bin/bash

function printHelpTextd
{
        echo
        echo "######################################################################"
        echo "#                                                                    #"
        echo "#  This script takes these parameters, in this order:                #"
        echo "#  1. check type, one of: certstartdate, certdaysleft, or certhash.  #"
        echo "#  2. Host connection target (IP address or host name (fqdn)).       #"
        echo "#  3. TCP port number to connect to.                                 #"
        echo "#                                                                    #"
        echo "#  This script returns, depending on the check type, one of:         #"
        echo "#  - certstartdate: a text string of the cert start date             #"
        echo "#  - certdaysleft: an integer of the number of days until the cert   #"
        echo "#    expiration; if the cert has expired, then a negative number.    #"
        echo "#  - certhash: a hash of the cert, useful for detecting changes.     #"
        echo "#                                                                    #"
        echo "######################################################################"
        echo
}


ERR_BADNUMPARAMS=1
ERR_BADCHECKTYPE=2

#  Function getCertStartDate
#  parameters (ordered)
#    * Host connection target (IP address or host name (fqdn)).
#    * TCP port number to connect to.
#  returns a text string of the certificate start date.
function getCertStartDate
{
        host=$1
        port=$2
        startdate=`echo quit | openssl s_client -host $host -port $port 2>/dev/null | awk '/BEGIN/{s=x}{s=s$0"\n"}/END CERTIFICATE-----/{print s}' 2>/dev/null | openssl x509 -noout -dates 2>/dev/null | head -n 1 | cut -d "=" -f 2- | awk -F " " '{ print $1" "$2" "$4" "$3" "$5 }'`
        echo $startdate
}


#  Function getCertDaysLeft
#  parameters (ordered)
#    * Host connection target (IP address or host name (fqdn)).
#    * TCP port number to connect to.
#  returns a number of days remaining
function getCertDaysLeft
{
        host=$1
        port=$2
        enddate=`echo quit | openssl s_client -host $host -port $port 2>/dev/null | awk '/BEGIN/{s=x}{s=s$0"\n"}/END CERTIFICATE-----/{print s}' | openssl x509 -noout -dates 2>/dev/null | tail -n 1 | cut -d "=" -f 2-`
        formattedenddate=`echo $enddate | awk -F " " '{ print $1" "$2" "$4" "$3" "$5 }'`
        enddateseconds=`date -d "$formattedenddate" +%s`
        # expiration date minus todays date = the number of days left (in seconds)
        secondsleft=$(expr $enddateseconds - $(date +%s))
        daysleft=$(expr $secondsleft / 86400)
        echo $daysleft
}


#  Function getCertHash
#  parameters (ordered)
#    * Host connection target (IP address or host name (fqdn)).
#    * TCP port number to connect to.
#  returns the hash of the cert, as a string
function getCertHash
{
        host=$1
        port=$2
        hash=`echo quit | openssl s_client -host $host -port $port 2>/dev/null | awk '/BEGIN/{s=x}{s=s$0"\n"}/END CERTIFICATE-----/{print s}' | openssl x509 -noout -hash 2>/dev/null`
        echo $hash
}


if [ "$#" -ne 3 ]; then
{
        echo "ERROR: Illegal number of parameters."
        printHelpText
        exit $ERR_BADNUMPARAMS
}; else
{
        Operation=$1
        TargetHost=$2
        TargetPort=$3
        case $Operation in
        certstartdate)
                getCertStartDate $TargetHost $TargetPort
                ;;
        certdaysleft)
                getCertDaysLeft $TargetHost $TargetPort
                ;;
        certhash)
                getCertHash $TargetHost $TargetPort
                ;;
        *)
                {
                        echo "ERROR: Bad check type."
                        printHelpText
                        exit $ERR_BADCHECKTYPE
                }
                ;;
        esac
}; fi

Automatically delete old NetApp snapshots left by backups

Backup software may sometimes leave behind a snapshot.  This can increase the space consumption a lot over time.

Here's a script that can be run against NetApp filers to clean up those "stale" backups.  Note that in my case, I configured CommVault to name the snapshots with the string "snapshot_for_backup", though the default is just "ndmp".  You may change that as needed.

#!/bin/bash

# This script is to look for snaps that are left over from backups, that # are no longer in use, and delete them.

# This script requires two parameters:

# snap_cleanup     (list or delete all stale snaps on the specified servers)

NasUser="root"

SnapshotString="snapshot_for_backup"

SshIdentityFile="/root/.ssh/id_rsa_auslxfs00_rsync"

SshBinary=/usr/bin/ssh

# This function receives a volume name as a parameter, and returns # the number of snaps that are elligible for deletion, defined by being:

#                             1. Having a certain string in the snapshot name;

#                             2. Not marked as "busy"

function CountStaleSnaps ()

{

                local VolToCheck=$1

                local StaleSnaps=`$SshCmd "snap list $VolToCheck" | grep $SnapshotString | grep -v "busy" | wc -l`

                if [ $? -ne 0 ]; then

                {

                                echo "getting count of elligible snapshots returned an error"

                                exit 1

                } else

                {

                                echo $StaleSnaps

                } fi

}

# Function GetStaleSnapNames

# This function creates, given a volume name, an array of snapshots that are # candidates for deletion.

# Parameters

#             1. volume to check

function GetStaleSnapNames ()

{

                local Volume=$1

                StaleSnapNames=( $($SshCmd "snap list $Volume" | grep $SnapshotString| grep -v busy | cut -c 39- | cut -f 1 -d " ") )

                if [ $? -ne 0 ]; then

                {

                                echo "getting names of elligible snapshots returned an error"

                                exit 1

                } else

                {

                                printf -- '%s\n' "${StaleSnapNames[@]}"

                } fi

}

# Function DelStaleSnaps

# This function deletes stale snaps for the Volume name passed to it.

# Parameters

#  1. volume

#  2. snapshot_name

function DelStaleSnaps ()

{

                local Volume=$1

                local SnapToDelete=$2

                $SshCmd "snap delete $Volume $SnapToDelete"

                if [ $? -ne 0 ]; then

                {

                                echo "error deleting snapshot $Volume:$SnapToDelete"

                                exit 1

                } else

                {

                                echo "successfully deleted snapshot $Volume:$SnapToDelete"

                } fi

}

# Parse command line parameters

case $2 in

                "")

                                echo "ERROR: you must specify as a second parameter a host name on which you want to delete snapshots."

                                ;;

                *)

                                NasName=$2

                                ;;

esac

case $1 in

                list)

                                Operation=list

                                ;;

                delete)

                                Operation=delete

                                ;;

                *)

                                echo "ERROR: invalid operation specified on command line.  Please specify either 'list' or 'delete' followed by the servername on which you want to delete the snapshots."

                                exit 1

                                ;;

esac

SshCmd="/usr/bin/ssh -i $SshIdentityFile $NasUser@$NasName"

VolumesToCheck=( `$SshCmd "vol status -b" | cut -f 1 -d " " | egrep -v "Volume|-----"` )

for CurrentVol in ${VolumesToCheck[@]}; do {

                echo -n "checking $NasName:$CurrentVol... "

                StaleSnaps=`CountStaleSnaps $CurrentVol`

                echo $StaleSnaps            

                if [ $Operation = list ]; then

                                if [ $StaleSnaps -ne 0 ]; then

                                                GetStaleSnapNames $CurrentVol |  awk '{ print "    " $1 }'

                                fi

                elif [ $Operation = delete ]; then

                                GetStaleSnapNames $CurrentVol

                                ArrayOfSnaps=( $(GetStaleSnapNames $CurrentVol) )

                                for TargetSnap in `printf -- '%s\n' "${ArrayOfSnaps[@]}"`; do

                                                                DelStaleSnaps $CurrentVol $TargetSnap

                                done

                fi

                               

}; done

Linux bash shell options parsing

Normally, command line parameters come in as separate, positional values, and may be referenced as:

• $# - the number of command line arguments (positional parameters)
• $* - all positional parameters expressed as a single string
• $@ - all positional parameters, but with each as a quoted string (each positional parameter is intact and presented as a quoted string)
• $0 - (the base name of the script itself)
• $1 - The first positional parameter
• $2 - The second positional parameter, and $3, $4, etc.  Starting with 10, they must be expressed as ${10}, ${11}, etc.

Sometimes a variable is passed from a wrapper script to a child script where the positional parameter is in fact several parameters that should be parsed separately.

Here, we test for that case, and if found, we peel off the first parameter within that group of strings, and assign the remaining parameters in the group of springs:

# from the front, and make the rest of the data be the options passed to rsync.
if [ $# -eq 1 ]; then
{
   Params=($1)
   PARAM_1=${Params[0]}
   unset Params[0]
   ChildCommandOpts=${Params[*]}
} fi

Check return code of piped command OR Export variable to parent shell

I just spent a day or more with a peer working the problem of "how do we get the return code of the first command in a series of piped commands in bash?"

The problem is: $? will hold the return code of the last command in the pipe sequence. We tried doing various things like ( foocmd ; export OUTERR=$?) | gzip... The problem with that is that, in bash, exported variables are not "global", so the value of $OUTERR is lost as soon as we hit the ). {}'s also did not work.

My partner's hack was to read stdout from foocmd into a variable, which he later cat'ed into gzip. Ick. Since we're dumping databases with foocmd, we're sure to run into architecture-dependent bash variable size limits, not to mention the RAM requirement of storing entire DB dumps into a variable.

As is usually the case, in the end we found that we'd spent so much time because we were going about it the wrong way, and we lacked the simple truth that would help us solve the problem efficiently.

${PIPESTATUS[@]} is an array similar to $?, except that it stores the return code of each component in a piped series. Thus, if we do this:

# /bin/false | tr x y | wc > /dev/null 2>&1

# echo ${PIPESTATUS[@]}

...we get as output...

1 0 0

...the first command, /bin/false returned "1", and the others returned "0" each.

Properly, we would test the value of each array element before considering the execution of the whole to be a success.

Now, that array will be overwritten/cleared the very next command, so the first thing we want to do is copy the array:

FOO_EXITCODE=("${PIPESTATUS[@]}")

So simple in the end.

Remote bulk file edits and administration with SSH and SED ( sed examples )

Want to deploy the zabbix agent to a bunch of Ubuntu Linux systems? Easy. But wait... the config file for each needs to be updated. How about this:

for target in host1 host2 host3 host4; do echo $target; ssh -t $target "apt-get -y install zabbix-agent; sed -i.bak -e \"s/Server=localhost/Server=10.10.1.11/g\" -e \"s/Hostname=localhost/Hostname=$target/g\" /etc/zabbix/zabbix_agent.conf /etc/zabbix/zabbix_agentd.conf; update-rc.d zabbix-agent enable; /etc/init.d/zabbix-agent restart"; done

This will:

1. ssh to each host
2. install the agent on that host
3. replace the default "Server=" and the "Hostname=" lines in the two config files zabbix_agent.conf and zabbix_agentd.conf", where 10.10.1.11 is the zabbix server ip address.
4. make a backup of the two config files
5. configure the zabbix_agent to auto-start
6. restart the zabbix agent to pick up the config file changes.

That was easier and more reliable than trying to complete the procedure on 50 systems.

(think about updating fstab and others for a mass of hosts.)

For a simple file in-place edit of one line of a file (such as to comment out a line on all the systems' config files):

for target in host1 host2 host3 host4; do echo $target; ssh $target "sed -i.bak -e 's/^domain mynisdomain server mynismaster.company.com$/g #domain mynisdomain server mynismaster.company.com' /etc/yp.conf"; done

If you have a file to edit, and the line you want to replace has quote marks, you'll need to escape them with \\\ like so:

for target in host1 host2 host3 host4; do ssh $target "sed -i.bak -e \"s/^ENABLED=\\\"false\\\"/ENABLED=\\\"true\\\"/g\" /etc/default/sysstat "; done

parallel xargs and Xen/VMware / performance / storage performance data gathering

I have a problem where certain virtual machines drop offline for brief periods of time.

Why? I don't know, but I suspect an I/O related issue (NFS / network hiccup). There's nothing in the logs.

How can I test this? Well, the built-in tools (sar/sysstat reports) Vcenter, or XenCenter don't keep historical data down to the second, so I will miss any hiccup that lasted only a few seconds. Furthermore, long-term (minute, 5 minute, etc) stats/graphs tend to average out the spikes, so I won't necessarily see a spike of a fine resolution (seconds), unless it's enough to register on a scale of a time resolution an order of magnitude greater.

I know that when local disk I/O is blocked, then iostat shows the cpu spending 100% in iowait, and an individual device queue maxes out (disk device may also go to 100% utilization, with no transactions or data actually read/written). So, how to get/keep historical data to compare against my Nagios alerts? How about gathering data with 1 second granularity, and then when an alert comes through, compare that timestamp against what I've found in the logs.

how to do that in a quick, parallel, automated fashion without manually logging in to 50 servers? Parallel execution with xargs and ssh:

First, distribute a public key for ssh to use, so you don't have to manually type the password on each box. Then,

echo xenhost1 xenhost2 xenhost3 vm1 vm2 vm3 vm4 vm5 | xargs -P 100 -d" " --replace="TARGET" bash -c 'ssh -i /root/.ssh/ssh_keyfile TARGET "iostat -xnt 1 || iostat -nt 1" > TARGET.log'

(the || is because Citrix XenServer 5.5.0 uses a different iostat version and syntax than the other RHEL/CentOS systems I have)

When I get an that "vm xyz is unavailable", I can look at the log file for each of the xen hosts, the nfs server, and the vm itself to see whether there was an I/O, cpu load, or other problem.

Also, once I have all the data, I can grep out lines that I care about and import them into excel to have trending data with 1 second granularity, e.g.:

egrep "nfsserver:/export/home" xenhost1.log > foo.csv

Of course, you could use this xargs and logfile method to snapshot many aspects of a system over time to troubleshoot a problem or for your own records (what process list, network connections, etc.), things that may not be caught by syslogd, dmesg, cacti, etc. I did something like this to capture total IOPS across our enterprise when we were planning to consolidate a bunch of local disk to a central NAS+SAN. Had to capture iops per-server over a period of a week. It made for a 30MB spreadsheet, but in the end I know my numbers were accurate for the peaks and valleys of our i/o load.

Parallelizing Tasks in Unix/Linux

From Ian C. Blenke, The easiest way is with parallelized xargs:

$ find . -name '*.jpg' | sed -e 's/.jpg$//' | xargs -P4 -l1 -i
convert {}.jpg {}.png

The -P flag for xargs is a _wonderful_ thing to learn. Do it now, it
will forever save you time. I use it daily in our huge farm of linux
servers, makes for far more bearable adminning.

Simple expect ssh example

This expect script would be called from a shell script, and would ssh to the host passed as an argument (argv), perform the command specified, and disconnect. (Thanks, Tiger O.)

#!/usr/bin/expect

set timeout 1
set cmd {uname -a}

spawn ssh root@$argv
expect_after eof { exit 0 }


## interact with SSH
expect "yes/no" { send "yes\r" }
expect "password:" { send "rootpasswd\r" }

expect "# "
send "$cmd\r"
expect "$cmd\r"
expect "(.*)\r"
send "exit\r

How to process command-line args in bash scripts

while [ $# -gt 0 ]; do
case $1 in
--somevar)
SOMEVAR=$2
SOMEVARSTR="--somevar $2"
shift; shift
;;
--othervar)
OTHERVAR=$2
shift; shift
;;
--version)
VERSION=$2
shift; shift
;;
--debug)
DEBUGSTR="--debug"
shift
;;
*)
if [ -n "$TARGET" -o ! -d $1 ]; then
usage
fi
TARGET=$1
shift
;;
esac
done

sudo: sudoers examples

Sudo can be used allow users to execute certain commands as other users (including root) on certain machines, with logging.

Edit the sudoers file with visudo. Note that to execute many system commands, your PATH will need to include /sbin:/usr/sbin

See what access is allowed with "sudo -l".

The best example file I found was at http://www.gratisoft.us/sudo/sample.sudoers , except that is uses "!", which is pointless (commands can be copied).

## Sample sudoers file ##
# *** Host_Alias specifications ***
# Host_Alias seems not to be useful, unless you have a
# global sudoers file that is replicated across multiple hosts.
# make LOCAL mean localhost (probably a bad idea, as this will allow it to run on any machine that has the sudoers file)
Host_Alias LOCAL = 127.0.0.1
# Anywhere that "LAN" is specified, these hosts apply:
Host_Alias LAN = ahost.mycompany.com, anotherhost.mycompany.com

# *** User_Alias specifications ***
# User_Alias allows you to group users. (better to use AD/NIS groups, for global/central management?)
# MAILADMINS user alias refers to users dick and jane
User_Alias MAILADMINS = dick, jane

# *** Runas_Alias specifications ***
# This specifies an alias or grouping of whom a command can be run as.
Runas_Alias SOMEONE = larry, tom

# *** Cmd_Alias specifications ***
# alias or group commands with full paths, to make things easier to read later.
Cmnd_Alias SU = /bin/su
Cmnd_Alias SMTP = /sbin/service postfix stop, /sbin/service postfix start, /sbin/service postfix status
Cmnd_Alias REBOOT = /usr/bin/reboot, /sbin/shutdown -r now

# *** Defaults specification ***
# make user john.doe not have to enter a password to run commands as another user
Default:john.doe nopasswd
# make user kate have no timeout, and add env variable "GOO" to the sudo environment, and run as linda by default, but always require the root password
Defaults:kate timestamp_timeout=-1, env_delete+="GOO", runas_default=linda, rootpw
# make user jim have to enter the password of whoever he's running a command as, every time, with 1 attempt allowed
Defaults:jim timestamp_timeout=0, runaspw, passwd_tries=1
# global defaults - log to a specific file.
Defaults logfile=/var/log/sudo.log, log_year

# *** User Privilege specification ***
# This is where we actually say who and where (as whom) can do what
# user/%group hostname = (user) command
# by default, root can do all commands as all users
root ALL=(ALL) ALL
# users jake and jim, on localhost, can execute crond without entering a password. (probably a bad idea)
jake,jim LOCAL = NOPASSWD: /sbin/service crond restart
# allow MAILADMINS on hosts LAN to run as root the commands SMTP and REBOOT.
MAILADMINS LAN = (root) SMTP, (SOMEONE) REBOOT
# members of the group "wheel" can run, on all hosts, all commands as all users
%wheel ALL=(ALL) ALL