| 1 | #!/bin/sh
|
|---|
| 2 |
|
|---|
| 3 | # This must run as root as CTDB tool commands need to access CTDB socket
|
|---|
| 4 | [ $(id -u) -eq 0 ] || exec sudo "$0" "$@"
|
|---|
| 5 |
|
|---|
| 6 | # this script needs to be installed so that statd points to it with the -H
|
|---|
| 7 | # command line argument. The easiest way to do that is to put something like this in
|
|---|
| 8 | # /etc/sysconfig/nfs:
|
|---|
| 9 | # STATD_HOSTNAME="myhostname -H /etc/ctdb/statd-callout"
|
|---|
| 10 |
|
|---|
| 11 | [ -n "$CTDB_BASE" ] || \
|
|---|
| 12 | export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
|
|---|
| 13 |
|
|---|
| 14 | . $CTDB_BASE/functions
|
|---|
| 15 |
|
|---|
| 16 | # Overwrite this so we get some logging
|
|---|
| 17 | die ()
|
|---|
| 18 | {
|
|---|
| 19 | script_log "statd-callout" "$@"
|
|---|
| 20 | exit 1
|
|---|
| 21 | }
|
|---|
| 22 |
|
|---|
| 23 | loadconfig ctdb
|
|---|
| 24 | loadconfig nfs
|
|---|
| 25 |
|
|---|
| 26 | [ -n "$NFS_HOSTNAME" ] || \
|
|---|
| 27 | die "NFS_HOSTNAME is not configured. statd-callout failed"
|
|---|
| 28 |
|
|---|
| 29 | # A handy newline
|
|---|
| 30 | nl="
|
|---|
| 31 | "
|
|---|
| 32 |
|
|---|
| 33 | ctdb_setup_service_state_dir "statd-callout"
|
|---|
| 34 |
|
|---|
| 35 | cd "$service_state_dir" || \
|
|---|
| 36 | die "Failed to change directory to \"${service_state_dir}\""
|
|---|
| 37 |
|
|---|
| 38 | case "$1" in
|
|---|
| 39 | # Keep a single file to keep track of the last "add-client" or
|
|---|
| 40 | # "del-client'. These get pushed to ctdb.tdb during "update",
|
|---|
| 41 | # which will generally be run once each "monitor" cycle. In this
|
|---|
| 42 | # way we avoid scalability problems with flood of persistent
|
|---|
| 43 | # transactions after a "notify" when all the clients re-take their
|
|---|
| 44 | # locks.
|
|---|
| 45 |
|
|---|
| 46 | add-client)
|
|---|
| 47 | # statd does not tell us to which IP the client connected so
|
|---|
| 48 | # we must add it to all the IPs that we serve
|
|---|
| 49 | cip="$2"
|
|---|
| 50 | ctdb_get_pnn
|
|---|
| 51 | date=$(date '+%s')
|
|---|
| 52 | ctdb ip -X |
|
|---|
| 53 | tail -n +2 |
|
|---|
| 54 | while IFS="|" read x sip node x ; do
|
|---|
| 55 | [ "$node" = "$pnn" ] || continue # not us
|
|---|
| 56 | key="statd-state@${sip}@${cip}"
|
|---|
| 57 | echo "\"${key}\" \"${date}\"" >"$key"
|
|---|
| 58 | done
|
|---|
| 59 | ;;
|
|---|
| 60 |
|
|---|
| 61 | del-client)
|
|---|
| 62 | # statd does not tell us from which IP the client disconnected
|
|---|
| 63 | # so we must add it to all the IPs that we serve
|
|---|
| 64 | cip="$2"
|
|---|
| 65 | ctdb_get_pnn
|
|---|
| 66 | ctdb ip -X |
|
|---|
| 67 | tail -n +2 |
|
|---|
| 68 | while IFS="|" read x sip node x ; do
|
|---|
| 69 | [ "$node" = "$pnn" ] || continue # not us
|
|---|
| 70 | key="statd-state@${sip}@${cip}"
|
|---|
| 71 | echo "\"${key}\" \"\"" >"$key"
|
|---|
| 72 | done
|
|---|
| 73 | ;;
|
|---|
| 74 |
|
|---|
| 75 | update)
|
|---|
| 76 | files=$(echo statd-state@*)
|
|---|
| 77 | if [ "$files" = "statd-state@*" ] ; then
|
|---|
| 78 | # No files!
|
|---|
| 79 | exit 0
|
|---|
| 80 | fi
|
|---|
| 81 | # Filter out lines for any IP addresses that are not currently
|
|---|
| 82 | # hosted public IP addresses.
|
|---|
| 83 | ctdb_get_pnn
|
|---|
| 84 | ctdb_ips=$(ctdb ip | tail -n +2)
|
|---|
| 85 | sed_expr=$(echo "$ctdb_ips" |
|
|---|
| 86 | awk -v pnn=$pnn 'pnn == $2 { \
|
|---|
| 87 | ip = $1; gsub(/\./, "\\.", ip); \
|
|---|
| 88 | printf "/statd-state@%s@/p\n", ip }')
|
|---|
| 89 | if cat $files | sed -n "$sed_expr" | ctdb ptrans "ctdb.tdb" ; then
|
|---|
| 90 | rm $files
|
|---|
| 91 | fi
|
|---|
| 92 | ;;
|
|---|
| 93 |
|
|---|
| 94 | notify)
|
|---|
| 95 | # we must restart the lockmanager (on all nodes) so that we get
|
|---|
| 96 | # a clusterwide grace period (so other clients don't take out
|
|---|
| 97 | # conflicting locks through other nodes before all locks have been
|
|---|
| 98 | # reclaimed)
|
|---|
| 99 |
|
|---|
| 100 | # we need these settings to make sure that no tcp connections survive
|
|---|
| 101 | # across a very fast failover/failback
|
|---|
| 102 | #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
|
|---|
| 103 | #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
|
|---|
| 104 | #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
|
|---|
| 105 |
|
|---|
| 106 | # Delete the notification list for statd, we don't want it to
|
|---|
| 107 | # ping any clients
|
|---|
| 108 | rm -f /var/lib/nfs/statd/sm/*
|
|---|
| 109 | rm -f /var/lib/nfs/statd/sm.bak/*
|
|---|
| 110 |
|
|---|
| 111 | # we must keep a monotonically increasing state variable for the entire
|
|---|
| 112 | # cluster so state always increases when ip addresses fail from one
|
|---|
| 113 | # node to another
|
|---|
| 114 | # We use epoch and hope the nodes are close enough in clock.
|
|---|
| 115 | # Even numbers mean service is shut down, odd numbers mean
|
|---|
| 116 | # service is started.
|
|---|
| 117 | state_even=$(( $(date '+%s') / 2 * 2))
|
|---|
| 118 |
|
|---|
| 119 | # We must also let some time pass between stopping and
|
|---|
| 120 | # restarting the lock manager. Otherwise there is a window
|
|---|
| 121 | # where the lock manager will respond "strangely" immediately
|
|---|
| 122 | # after restarting it, which causes clients to fail to reclaim
|
|---|
| 123 | # their locks.
|
|---|
| 124 | "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
|
|---|
| 125 | sleep 2
|
|---|
| 126 | "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
|
|---|
| 127 |
|
|---|
| 128 | # we now need to send out additional statd notifications to ensure
|
|---|
| 129 | # that clients understand that the lockmanager has restarted.
|
|---|
| 130 | # we have three cases:
|
|---|
| 131 | # 1, clients that ignore the ip address the stat notification came from
|
|---|
| 132 | # and ONLY care about the 'name' in the notify packet.
|
|---|
| 133 | # these clients ONLY work with lock failover IFF that name
|
|---|
| 134 | # can be resolved into an ipaddress that matches the one used
|
|---|
| 135 | # to mount the share. (==linux clients)
|
|---|
| 136 | # This is handled when starting lockmanager above, but those
|
|---|
| 137 | # packets are sent from the "wrong" ip address, something linux
|
|---|
| 138 | # clients are ok with, buth other clients will barf at.
|
|---|
| 139 | # 2, Some clients only accept statd packets IFF they come from the
|
|---|
| 140 | # 'correct' ip address.
|
|---|
| 141 | # 2a,Send out the notification using the 'correct' ip address and also
|
|---|
| 142 | # specify the 'correct' hostname in the statd packet.
|
|---|
| 143 | # Some clients require both the correct source address and also the
|
|---|
| 144 | # correct name. (these clients also ONLY work if the ip addresses
|
|---|
| 145 | # used to map the share can be resolved into the name returned in
|
|---|
| 146 | # the notify packet.)
|
|---|
| 147 | # 2b,Other clients require that the source ip address of the notify
|
|---|
| 148 | # packet matches the ip address used to take out the lock.
|
|---|
| 149 | # I.e. that the correct source address is used.
|
|---|
| 150 | # These clients also require that the statd notify packet contains
|
|---|
| 151 | # the name as the ip address used when the lock was taken out.
|
|---|
| 152 | #
|
|---|
| 153 | # Both 2a and 2b are commonly used in lockmanagers since they maximize
|
|---|
| 154 | # probability that the client will accept the statd notify packet and
|
|---|
| 155 | # not just ignore it.
|
|---|
| 156 | # For all IPs we serve, collect info and push to the config database
|
|---|
| 157 | ctdb_get_pnn
|
|---|
| 158 |
|
|---|
| 159 | # Construct a sed expression to take catdb output and produce pairs of:
|
|---|
| 160 | # server-IP client-IP
|
|---|
| 161 | # but only for the server-IPs that are hosted on this node.
|
|---|
| 162 | ctdb_all_ips=$(ctdb ip all | tail -n +2)
|
|---|
| 163 | sed_expr=$(echo "$ctdb_all_ips" |
|
|---|
| 164 | awk -v pnn=$pnn 'pnn == $2 { \
|
|---|
| 165 | ip = $1; gsub(/\./, "\\.", ip); \
|
|---|
| 166 | printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
|
|---|
| 167 |
|
|---|
| 168 | statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort)
|
|---|
| 169 | [ -n "$statd_state" ] || exit 0
|
|---|
| 170 |
|
|---|
| 171 | smnotify="${CTDB_HELPER_BINDIR}/smnotify"
|
|---|
| 172 | prev=""
|
|---|
| 173 | echo "$statd_state" | {
|
|---|
| 174 | # This all needs to be in the same command group at the
|
|---|
| 175 | # end of the pipe so it doesn't get lost when the loop
|
|---|
| 176 | # completes.
|
|---|
| 177 | items=""
|
|---|
| 178 | while read sip cip ; do
|
|---|
| 179 | # Collect item to delete from the DB
|
|---|
| 180 | key="statd-state@${sip}@${cip}"
|
|---|
| 181 | item="\"${key}\" \"\""
|
|---|
| 182 | items="${items}${items:+${nl}}${item}"
|
|---|
| 183 |
|
|---|
| 184 | # NOTE: Consider optimising smnotify to read all the
|
|---|
| 185 | # data from stdin and then run it in the background.
|
|---|
| 186 |
|
|---|
| 187 | # Reset stateval for each serverip
|
|---|
| 188 | [ "$sip" = "$prev" ] || stateval="$state_even"
|
|---|
| 189 | # Send notifies for server shutdown
|
|---|
| 190 | "$smnotify" --client=$cip --ip=$sip --server=$sip --stateval=$stateval
|
|---|
| 191 | "$smnotify" --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
|
|---|
| 192 | # Send notifies for server startup
|
|---|
| 193 | stateval=$(($stateval + 1))
|
|---|
| 194 | "$smnotify" --client=$cip --ip=$sip --server=$sip --stateval=$stateval
|
|---|
| 195 | "$smnotify" --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval
|
|---|
| 196 | done
|
|---|
| 197 |
|
|---|
| 198 | echo "$items" | ctdb ptrans "ctdb.tdb"
|
|---|
| 199 | }
|
|---|
| 200 |
|
|---|
| 201 | # Remove any stale touch files (i.e. for IPs not currently
|
|---|
| 202 | # hosted on this node and created since the last "update").
|
|---|
| 203 | # There's nothing else we can do with them at this stage.
|
|---|
| 204 | echo "$ctdb_all_ips" |
|
|---|
| 205 | awk -v pnn=$pnn 'pnn != $2 { print $1 }' |
|
|---|
| 206 | while read sip ; do
|
|---|
| 207 | rm -f "statd-state@${sip}@"*
|
|---|
| 208 | done
|
|---|
| 209 | ;;
|
|---|
| 210 | esac
|
|---|
