condor_configd

#!/usr/bin/python
#   Copyright 2013 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import os
import sys
import signal
import time
import socket
import threading
import getopt
import random
import tempfile
import shutil
import stat
import wallaroo_config
from condorutils.log import *
from condorutils.readconfig import *

class Timer (threading.Thread):
  def __init__(self, interval, function, name=None, args=[], kwargs={}):
    threading.Thread.__init__(self)
    if name != None:
      self.name = name
    self.interval = interval
    self.function = function
    self.args = args
    self.kwargs = kwargs
    self.setDaemon(True)
    self.finished = threading.Event()


  def stop(self):
    self.finished.set()
    self.join()


  def run(self):
    while not self.finished.isSet():
      self.finished.wait(self.interval)
      if not self.finished.isSet():
        self.function(*self.args, **self.kwargs)

class Configd:
  def __init__(self):
    self.sysinfo = {}
    self.connect_opts = {"tag":"current"}
    self.checkin_interval = 0
    self.shutdown_interval = 0
    self.sysinfo_interval = 0
    self.log_name = os.path.basename(sys.argv[0])
    self.node_name = socket.gethostname()
    self.managedfile = ""
    self.config_dir = ""
    self.override_dir = ""
    self.old_config = ""
    self.opt_user = ""
    self.opt_passwd = ""
    self.opt_file = ""
    self.log_file = ""
    self.log_level = logging.INFO
    self.pidfile = os.path.normpath(os.getcwd() + "/.pid" + str(os.getpid()))
    self.replacing_file = False
    self.stop_running = False
    self.retrieve = False
    self.notifications = False
    self.periodic_timer = None
    self.shutdown_timer = None
    self.sysinfo_timer = None
    self.broker_connection = None
    self.session = None
    self.receiver = None
    self.address = "mrg.grid.config.notifications"
    self.old_config_filename = "condor.old_config"
    self.periodic_thread = "Thread for Interval Timer"
    self.shutdown_thread = "Thread for Windows Shutdown Timer"
    self.sysinfo_thread = "Thread for System Info Timer"


  def set_handlers(self):
    # Set signal handlers
    signal.signal(signal.SIGINT, self.start_shutdown)
    signal.signal(signal.SIGTERM, self.start_shutdown)
    signal.signal(signal.SIGABRT, self.start_shutdown)
    signal.signal(signal.SIGILL, self.start_shutdown)
    signal.signal(signal.SIGFPE, self.start_shutdown)
    signal.signal(signal.SIGSEGV, self.start_shutdown)
    if not self.is_windows():
       # These aren't available on windows
       signal.signal(signal.SIGQUIT, self.start_shutdown)
       signal.signal(signal.SIGHUP, self.reconfig)


  def parse_args(self):
    long_opts = ["debug", "hostname=", "logfile=", "managedfile=", 
                 "password=", "retrieve", "user="]

    try:
      opts, args = getopt.gnu_getopt(sys.argv[1:], "dh:l:m:rP:U:", long_opts)
    except getopt.GetoptError, error:
      print str(error)
      return(1)

    for option, arg in opts:
      if option in ("-d", "--debug"):
        self.log_level = logging.DEBUG
      if option in ("-h", "--hostname"):
        self.node_name = arg.strip()
      if option in ("-l", "--logfile"):
        self.log_file = arg.strip()
      if option in ("-m", "--managedfile"):
        self.opt_file = arg.strip()
      if option in ("-P", "--password"):
        self.opt_passwd = arg.strip()
      if option in ("-r", "--retrieve"):
        self.retrieve = True
      if option in ("-U", "--user"):
        self.opt_user = arg.strip()


  def init(self):
    self.set_handlers()
    self.parse_args()

    # Configure the logging system
    if self.log_file == "":
      try:
        self.log_file = read_condor_config("CONFIGD", ["LOG"], permit_param_only = False)["log"]
      except ConfigError, error:
        print "Error: %s.  Exiting" % error.msg
        return(1)

    try:
      size = int(read_condor_config("", ["MAX_CONFIGD_LOG"])["max_configd_log"])
    except:
      size = 1000000

    # Create the log file
    try:
      create_file_logger(self.log_name, self.log_file, self.log_level, size=size)
    except Exception, e:
      print e
      print "Failed to open log file.  Exiting"
      return(1)


  def config_store(self):
    user = self.opt_user
    password = self.opt_passwd

    try:
      host = read_condor_config("CONFIGD", ["STORE_HOST"])["store_host"]
    except ConfigError, error:
      log(logging.WARNING, self.log_name, error.msg)
    except Exception, e:
      log(logging.WARNING, self.log_name, e)

    try:
      self.connect_opts["host"] = host
    except:
      log(logging.WARNING, self.log_name, "Using default (127.0.0.1)")
      self.connect_opts["host"] = "127.0.0.1"

    try:
      port = read_condor_config("CONFIGD", ["STORE_PORT"])["store_port"]
    except ConfigError, error:
      log(logging.DEBUG, self.log_name, error.msg)
    except Exception, e:
      log(logging.DEBUG, self.log_name, e)

    try:
      self.connect_opts["port"] = int(port)
    except:
      log(logging.DEBUG, self.log_name, "Invalid or undefined port.  Using default (8000)")
      self.connect_opts["port"] = 8000

    if self.opt_user == "":
      try:
        user = str(read_condor_config("CONFIGD", ["STORE_USERNAME"])["store_username"])
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)

    if user == "":
      log(logging.INFO, self.log_name, "Authenticating to the store as default user")
      user = None
    else:
      log(logging.INFO, self.log_name, "Authenticating to the store as user %s" % user)
    self.connect_opts["username"] = user

    if self.opt_passwd == "":
      try:
        pwfile = str(read_condor_config("CONFIGD", ["STORE_PASSWORD_FILE"])["store_password_file"])
        fobj = open(pwfile, 'r')
        password = fobj.read().strip()
        fobj.close()
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)

    if password == "":
      log(logging.INFO, self.log_name, "Will not supply a password for store authentication")
      password = None
    else:
      log(logging.INFO, self.log_name, "Will supply a password for store authentication")
    self.connect_opts["pw"] = password


  def config_broker(self):
    methods = "ANONYMOUS"
    host = "127.0.0.1"
    user = ""
    password = ""
    port = 5672

    if self.broker_connection != None:
      self.broker_connection.close()
      self.broker_connection = None

    try:
      self.notifications = (read_condor_config("CONFIGD", ["ENABLE_NOTIFICATIONS"], permit_param_only = False)["enable_notifications"].upper() == "TRUE")
    except ConfigError, error:
      log(logging.DEBUG, self.log_name, error.msg)
    except Exception, e:
      log(logging.DEBUG, self.log_name, e)

    if self.notifications == False:
      log(logging.INFO, self.log_name, "Receiving configuration notifications disabled")
    else:
      try:
        from qpid.messaging import Connection
      except:
        self.notifications = False
        log(logging.INFO, self.log_name, "Can't find notification support libraries.  This node will not receive configuration notifications")
        return

      log(logging.INFO, self.log_name, "Receiving configuration notifications enabled")
      try:
        host = read_condor_config("CONFIGD", ["BROKER_HOST"], permit_param_only = False)["broker_host"]
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)
      log(logging.DEBUG, self.log_name, "Connecting to broker at %s" % host)

      try:
        port = int(read_condor_config("CONFIGD", ["BROKER_PORT"], permit_param_only = False)["broker_port"])
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)
      log(logging.DEBUG, self.log_name, "Using port %d to connect to the broker" % port)

      try:
        methods = str(read_condor_config("CONFIGD", ["BROKER_AUTH_MECH"])["broker_auth_mech"]).replace(',', ' ')
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)
      log(logging.DEBUG, self.log_name, "Connecting with authentication methods \"%s\"" % methods)

      try:
        user = str(read_condor_config("CONFIGD", ["BROKER_USERNAME"])["broker_username"])
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)

      if user == "":
        log(logging.INFO, self.log_name, "Authenticating to broker as default user")
      else:
        log(logging.INFO, self.log_name, "Authenticating to broker as user %s" % user)

      try:
        pwfile = str(read_condor_config("CONFIGD", ["BROKER_PASSWORD_FILE"])["broker_password_file"])
        fobj = open(pwfile, 'r')
        password = fobj.read().strip()
        fobj.close()
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)

      if password == "":
        log(logging.INFO, self.log_name, "Will not supply a password for broker authentication")
      else:
        log(logging.INFO, self.log_name, "Will supply a password for broker authentication")

      try:
        self.address = str(read_condor_config("CONFIGD", ["BROKER_ADDRESS"])["broker_address"])
      except ConfigError, error:
        log(logging.DEBUG, self.log_name, error.msg)
      except Exception, e:
        log(logging.DEBUG, self.log_name, e)
      log(logging.DEBUG, self.log_name, "Listening for notifications from address %s" % self.address)

      self.broker_connection = Connection("%s:%s" % (host, port), username=user, password=password, sasl_mechanisms=methods, reconnect=True)
      self.broker_connection.open()
      self.session = self.broker_connection.session()


  def config(self):
    filename = self.opt_file

    # Store connection params
    self.config_store()

    # Broker params for notifications
    self.config_broker()

    try:
      self.checkin_interval = int(read_condor_config("CONFIGD", ["CHECK_INTERVAL"], permit_param_only = False)["check_interval"])
    except ConfigError, error:
      log(logging.INFO, self.log_name, error.msg)
    except Exception, e:
      log(logging.INFO, self.log_name, e)

    if self.checkin_interval <= 0:
      log(logging.INFO, self.log_name, "Invalid CHECK_INTERVAL. Using default (3600)")
      self.checkin_interval = 3600

    if self.is_windows():
      try:
        self.shutdown_interval = int(read_condor_config("CONFIGD", ["WIN_INTERVAL"], permit_param_only = False)["win_interval"])
      except ConfigError, error:
        log(logging.WARNING, self.log_name, error.msg)
      except:
        log(logging.WARNING, self.log_name, "WIN_INTERVAL must be a valid integer value")

      # Ensure there is a valid interval above 0 as this timer can not be
      # disabled
      if self.shutdown_interval <= 0:
        log(logging.WARNING, self.log_name, "Using default WIN_INTERVAL (5)")
        self.shutdown_interval = 5

    try:
      self.sysinfo_interval = int(read_condor_config("CONFIGD", ["UPDATE_INTERVAL"], permit_param_only = False)["update_interval"])
    except ConfigError, error:
      log(logging.INFO, self.log_name, error.msg)
    except Exception, e:
      log(logging.INFO, self.log_name, e)

    if self.sysinfo_interval < 0:
      log(logging.INFO, self.log_name, "Invalid UPDATE_INTERVAL. Using default (10)")
      self.sysinfo_interval = 10

    if self.opt_file == "":
      try:
        filename = read_condor_config("", ["LOCAL_CONFIG_FILE"])["local_config_file"]
      except ConfigError, error:
        raise RuntimeError(error.msg + ". Unable to write to configuration file")
      except Exception, e:
        raise RuntimeError(e + ". Invalid value for LOCAL_CONFIG_FILE")
    self.managedfile = os.path.normpath(filename)
    log(logging.DEBUG, self.log_name, "Writing configuration to \"%s\"" % self.managedfile)

    try:
      self.config_dir = read_condor_config("", ["LOCAL_CONFIG_DIR"])["local_config_dir"]
      if os.access(self.config_dir, os.F_OK) != True or os.access(self.config_dir, os.R_OK) != True:
        raise RuntimeError("\"%s\" either is not a directory or has incorrect permissions." % self.config_dir)
    except ConfigError, error:
      raise RuntimeError(error.msg + ". Unable to read configuration files")
    except Exception, e:
      raise RuntimeError(e + ". Invalid value for LOCAL_CONFIG_DIR")

    try:
      self.override_dir = read_condor_config("CONFIGD", ["OVERRIDE_DIR"], permit_param_only = False)["override_dir"]
      if os.access(self.override_dir, os.F_OK) != True or os.access(self.override_dir, os.R_OK) != True:
        log(logging.WARNING, self.log_name, "\"%s\" either is not a directory or has incorrect permissions.  Configuration overrides will not take effect" % self.override_dir)
        self.override_dir = ""
    except:
      pass


  def reconfig(self, signum, frame):
    log(logging.DEBUG, self.log_name, "Received SIGHUP.  Reconfiguring")
    try:
      self.config()
    except Exception, e:
      log(logging.ERROR, self.log_name, e)
      self.start_shutdown(signal.SIGTERM, None)
      return
    self.clear_interval_timer()
    self.clear_shutdown_timer()
    self.clear_sysinfo_timer()
    self.setup_timers()


  def is_windows(self):
    return(os.name == "nt" or os.name == "ce")


  def get_node_obj(self):
    opts = self.connect_opts
    opts["node"] = self.node_name
    try:
      obj = wallaroo_config.connect(**opts)
    except Exception, e:
      if self.stop_running == False:
        log(logging.ERROR, self.log_name, "Exception retrieving node object")
      return(None)
    return(obj)


  def connect_receiver(self):
    try:
      self.receiver = self.session.receiver(self.address)
    except Exception, e:
      log(logging.DEBUG, self.log_name, e)
      self.receiver = None


  def clear_interval_timer(self):
    if self.periodic_timer != None:
      self.periodic_timer.stop()
      del self.periodic_timer
    self.periodic_timer = None


  def clear_shutdown_timer(self):
    if self.shutdown_timer != None:
      self.shutdown_timer.stop()
      del self.shutdown_timer
    self.shutdown_timer = None


  def clear_sysinfo_timer(self):
    if self.sysinfo_timer != None:
      self.sysinfo_timer.stop()
      del self.sysinfo_timer
    self.sysinfo_timer = None


  def setup_timers(self):
    # Setup the timer for checking configuration version
    if self.periodic_timer == None:
      self.periodic_timer = Timer(self.checkin_interval, self.check_config_ver, self.periodic_thread)
      self.periodic_timer.start()

    # Set a timer for shutdown on windows
    if self.is_windows() and self.shutdown_timer == None:
      self.shutdown_timer = Timer(self.shutdown_interval, self.test_for_shutdown, self.shutdown_thread)
      self.shutdown_timer.start()

    if self.sysinfo_timer == None and self.sysinfo_interval > 0:
      self.sysinfo_timer = Timer(self.sysinfo_interval, self.update_metadata, self.sysinfo_thread)
      self.sysinfo_timer.start()


  def monitor_timers(self):
    interval_found = False
    shutdown_found = False
    sysinfo_found = False

    for thread in threading.enumerate():
      if thread.name == self.periodic_thread:
        interval_found = True

      if thread.name == self.sysinfo_thread:
        sysinfo_found = True

      if self.is_windows() and thread.name == self.shutdown_thread:
        shutdown_found = True

    # Start the timers that are expected to be running but aren't
    if interval_found != True and self.checkin_interval > 0:
      log(logging.DEBUG, self.log_name, "Restarting interval timer")
      self.clear_interval_timer()

    if self.is_windows() and shutdown_found != True:
      log(logging.DEBUG, self.log_name, "Restarting windows shutdown timer")
      self.clear_shutdown_timer()

    if sysinfo_found != True and self.sysinfo_interval > 0:
      log(logging.DEBUG, self.log_name, "Restarting system info updating timer")
      self.clear_sysinfo_timer()

    self.setup_timers()


  def test_for_shutdown(self):
    log(logging.DEBUG, self.log_name, "Testing for shutdown file")
    if os.path.isfile(self.pidfile):
      log(logging.DEBUG, self.log_name, "Found shutdown file")
      os.remove(self.pidfile)
      start_shutdown(signal.SIGTERM, None)


  def start_shutdown(self, signum, frame):
    if self.stop_running != True:
      log(logging.DEBUG, self.log_name, "Shutting down")
      self.clear_interval_timer()
      self.clear_shutdown_timer()
      log(logging.DEBUG, self.log_name, "Setting stop flag")
    self.stop_running = True


  def shutdown(self):
    self.start_shutdown(signal.SIGTERM, None)
    log(logging.INFO, self.log_name, "Exiting")
    logging.shutdown()


  def check_config_ver(self, ver=0, force=False):
    log(logging.DEBUG, self.log_name, "Checking version of configuration")
    node = self.get_node_obj()
    if node == None:
      return(False)

    # If the passed version is 0, look at the last_updated
    if ver == 0:
      ver = node.config()["WALLABY_CONFIG_VERSION"]

    # Retrieve and install the node's configuration (if needed)
    ret = self.get_config(ver, force)
    if os.path.exists(self.old_config) == True:
      os.remove(self.old_config)
    return(ret)


  def get_config(self, version="", force=False):
    ignore_params = ["WALLABY_FORCE_CONFIG_PULL", "WALLABY_FORCE_RESTART"]

    node_obj = self.get_node_obj()
    if node_obj == None:
      return(False)

    # Check in with the store
    log(logging.DEBUG, self.log_name, "Performing a checkin with the store")
    try:
      node_obj.checkin()
    except Exception, error:
      if self.stop_running == False:
        log(logging.ERROR, self.log_name, error)
        log(logging.ERROR, self.log_name, "Failed to check in with the store")
      return(False)
    log(logging.DEBUG, self.log_name, "Checked in with the store")

    # Get the current WALLABY_CONFIG_VERSION.  If the system is running the
    # version as what was received, then there's no need to do anything
    (retval, running_version, err) = run_cmd("condor_config_val WALLABY_CONFIG_VERSION")
    try:
      running_version = running_version.strip()
    except:
      running_version = ""
 
    if version == running_version and force == False:
      log(logging.DEBUG, self.log_name, "The system is already running configuration version \"%s\"" % version)
      return(True)

    log(logging.INFO, self.log_name, "Retrieving configuration version \"%s\" from the store" % version)
   
    # Save the current configuraton
    (file_hdl, self.old_config) = tempfile.mkstemp(".tmp", self.old_config_filename, text=True)
    (retval, config_dump, err) = run_cmd("condor_config_val -dump")

    # Write the config from the dump
    for line in config_dump.split('\n'):
      if "local_config_file" not in line.lower() and \
         "local_config_dir" not in line.lower():
        os.write(file_hdl, "%s\n" % line)
    os.close(file_hdl)

    # Retrieve the node's configuration
    try:
      config = node_obj.config()
    except Exception, error:
      # Something has gone away
      if self.stop_running == False:
        log(logging.ERROR, self.log_name, error)
        log(logging.ERROR, self.log_name, "Exception when attempting to retrieve configuration from the store")
      return(False)

    try:
      (file_hdl,file_name) = tempfile.mkstemp(".tmp", "condor_config.local", text=True)
      # Write the config from the store into the file
      for key in config.keys():
        if key.strip().lower() in ignore_params:
          continue
        if key.strip().lower() == "dc_daemon_list":
          os.write(file_hdl, "%s =+ %s\n" % (key, config[key]))
        else:
          os.write(file_hdl, "%s = %s\n" % (key, config[key]))

      # Add the wallaby group/feature information
      print_str = ""
      log(logging.DEBUG, self.log_name, node_obj.features())
      for feat in node_obj.features():
        print_str += "%s," % feat
      os.write(file_hdl, "WALLABYFEATURES = \"%s\"\n" % print_str[:-1])

      print_str = ""
      for group in node_obj.memberships():
        print_str += "%s," % group
      os.write(file_hdl, "WALLABYGROUPS = \"%s\"\n" % print_str[:-1])
      os.write(file_hdl, "MASTER_ATTRS = $(MASTER_ATTRS), WallabyFeatures, WallabyGroups\n")
      os.write(file_hdl, "STARTD_ATTRS = $(STARTD_ATTRS), WallabyFeatures, WallabyGroups\n")

      # Now append the configuration from the configd configuration file
      cfg_name = os.path.normpath("%s/99configd.config" % self.config_dir)
      cfg_file = open(cfg_name, 'r')
      for line in cfg_file:
        os.write(file_hdl, line)
      cfg_file.close()

      # Process any overriden parameters
      if self.override_dir != "":
        entries = os.listdir(self.override_dir)
        entries.sort()
        for name in entries:
          fname = os.path.normpath("%s/%s" % (self.override_dir, name))
          if os.access(fname, os.R_OK) == True:
            try:
              hdl = open(fname, 'r')
              os.write(file_hdl, "# Override from \"%s\"\n" % fname)
              for line in hdl:
                os.write(file_hdl, line)
            except:
              log(logging.ERROR, self.log_name, "Problem reading file \"%s\"" % fname)
              hdl.close()
          else:
            log(logging.ERROR, self.log_name, "Unable to access \"%s\".  Ignoring override file" % fname)

      # Ensure permissions for restart/reconfig
      try:
        subs = node_obj.all_subsystems()
      except Exception, error:
        log(logging.WARNING, self.log_name, error)
        log(logging.WARNING, self.log_name, "Failed to retrieve subsystem list.  Configuration could break restart/reconfig functionality")
        subs = []

      dlist = []
      if "DAEMON_LIST" in config.keys():
        for d in config["DAEMON_LIST"].split(','):
          d = d.strip()
          if d != "":
            dlist += [d.upper()]

        for sub in subs:
          name = sub.upper()
          if name in dlist:
            os.write(file_hdl, "%s.SEC_ADMINISTRATOR_AUTHENTICATION_METHODS = $(%s.SEC_ADMINISTRATOR_AUTHENTICATION_METHODS), FS, NTLM, CLAIMTOBE\n" % (name, name))
      os.close(file_hdl)
    except IOError, e:
      log(logging.ERROR, self.log_name, e)
      try:
        os.close(file_hdl)
        os.remove(file_name)
      except:
        pass
      log(logging.ERROR, self.log_name, "Failed to read configd configuration/override file(s).  Not writing configuration file")
      return(False)
    except Exception, e:
      log(logging.ERROR, self.log_name, e)
      try:
        os.close(file_hdl)
        os.remove(file_name)
      except Exception, e:
        log(logging.ERROR, self.log_name, e)
        pass
      log(logging.ERROR, self.log_name, "Failed to write configuration to temp file")
      return(False)
   
    # Verify the config file is valid
    (retval, out, err) = run_cmd("condor_config_val -dump", environ={"CONDOR_CONFIG":"%s" % file_name})
    if retval != 0:
      log(logging.ERROR, self.log_name, "Configuration is invalid.  Discarding")
      os.remove(self.old_config)
      os.remove(file_name)
      return(False)
   
    # Install the file for condor to use
    self.replacing_file = True
    if os.path.exists(self.managedfile):
      os.remove(self.managedfile)
    try:
      shutil.move(file_name, self.managedfile)
    except Exception, e:
      log(logging.ERROR, self.log_name, e)
      log(logging.ERROR, self.log_name, "Error installing new configuration file")
      os.remove(self.old_config)
      os.remove(file_name)
      self.replacing_file = False
      return(False)

    if self.is_windows():
      try:
        (retval, out, err) = run_cmd("icacls %s /grant Everyone:(R)" % self.managedfile)
        if retval != 0:
          log(logging.ERROR, self.log_name, "Failed to set read permissions on \"%s\" (stdout = %s, stderr = %s)" % (self.managedfile, out, err))
      except Exception, e:
        # The command exists, and is needed, on newer versions of
        # windows that need to have the permissions changed.  If it
        # fails or doesn't exist, log data and move on
        log(logging.WARNING, self.log_name, e)
        log(logging.WARNING, self.log_name, "Problem running icacls to set permissions")
    else:
      os.chmod(self.managedfile, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
    self.replacing_file = False
    log(logging.INFO, self.log_name, "Retrieved configuration from the store")
 
    # Don't attempt to restart/reconfig anything if the config
    # retrieval was forced
    if force == True:
      return(True)

    # Have the store tell us which subsystems to restart/reconfig
    try:
      params, restart_list, reconfig_list = node_obj.whatChanged(running_version, version)
    except Exception, e:
      if self.stop_running == False:
        log(logging.ERROR, self.log_name, e)
        log(logging.ERROR, self.log_name, "Unable to retrieve configuration differences between \"%s\" and \"%s\".  The configuration will not take effect" % (running_version, version))
      return(False)

    log(logging.DEBUG, self.log_name, "Daemons to restart: %s" % restart_list)
    log(logging.DEBUG, self.log_name, "Daemons to reconfig: %s" % reconfig_list)

    # Determine the list of daemons the master is running, and only send
    # events to these daemons.  Any new daemons that should be running (or
    # any daemons that should be stopped) will be handled by commands sent
    # to the master
    (retval, daemons, err) = run_cmd("condor_config_val -master DAEMON_LIST")
    daemon_list = []
    if daemons != None:
      for daemon in daemons.split(','):
        daemon = daemon.strip()
        if daemon != "":
          daemon_list += [daemon.lower()]

    # Process the subsystem lists and act upon them.  Start by processing
    # the daemons to restart, and if master is listed then don't process
    # anything else because restarting the master will take care of
    # everything
    cmd = "condor_restart"
    if "master" in restart_list:
      return(self.send_condor_command(cmd, "master"))
    else:
      success = self.act_upon_subsys_list(cmd, restart_list, daemon_list)
   
      # Send a reconfig to the master daemon so all subsystems are
      # running with the current config values
      return(self.send_condor_command("condor_reconfig", "master") and success)


  def act_upon_subsys_list(self, command, slist, running_daemons):
    nondc_daemons = ["ll_daemon", "configd"]
    all_success = True

    for subsys in slist:
      subsys = str(subsys.strip())
      if subsys.lower() not in running_daemons:
        log(logging.DEBUG, self.log_name, "Not sending \"%s\" to subsystem \"%s\" since it is not currently running" % (command, subsys))
      else:
        if subsys.lower() in nondc_daemons:
          # Non-daemoncore daemon, so send an off first then an on
          # command.  It's possible the daemon isn't running for some
          # reason, and that is ok.  The important thing is to ensure
          # it is started
          self.send_condor_command("condor_off", subsys)
          if self.send_condor_command("condor_on", subsys) == False:
            all_success = False
          else:
            if self.send_condor_command(command, subsys) == False:
              all_success = False
    return(all_success)


  def send_condor_command(self, command, subsystem):
    log(logging.DEBUG, self.log_name, "Sending command \"%s\" to subsystem \"%s\"" % (command, subsystem))
    (retval, out, err) = run_cmd("%s -subsystem %s" % (command, subsystem), environ={"CONDOR_CONFIG":"%s" % self.old_config})
    if retval != 0:
      log(logging.ERROR, self.log_name, "Failed to send command \"%s\" to subsystem \"%s\" (retval: %d, stdout: \"%s\", stderr: \"%s\")" % (command, subsystem, retval, out, err))
      return(False)
    else:
      log(logging.DEBUG, self.log_name, "Sent command \"%s\" to subsystem \"%s\"" % (command, subsystem))
      return(True)


  def check_for_notification(self):
    from qpid.messaging import Empty

    msg = None

    if self.receiver == None:
      self.connect_receiver()

    if self.receiver != None:
      try:
        msg = self.receiver.fetch(timeout=0)
      except Empty:
        # No notification
        pass
      except Exception, e:
        # We lost our connection so force recreation
        log(logging.WARNING, self.log_name, e)
        self.receiver = None

    if msg != None:
      log(logging.DEBUG, self.log_name, "Received a configuration change notification")
      if self.node_name in msg.content["nodes"] or '*' in msg.content["nodes"]:
        log(logging.DEBUG, self.log_name, "The notification is for this node")
#        ver = msg.content["version"]
#        if self.check_config_ver(ver) == False:
        if self.check_config_ver() == False:
#          log(logging.ERROR, self.log_name, "Failed to install configuration version \"%s\"" % ver)
          log(logging.ERROR, self.log_name, "Failed to install new configuration")
      else:
        log(logging.DEBUG, self.log_name, "The notification is not for this node")
      self.session.acknowledge()


  def loadavg(self):
    (retval, lavg, err) = run_cmd("cat /proc/loadavg")
    data = lavg.split()
    self.sysinfo["loadAverage1Min"] = data.pop(0).strip()
    self.sysinfo["loadAverage5Min"] = data.pop(0).strip()
    self.sysinfo["loadAverage10Min"] = data.pop(0).strip()
    procs = data.pop(0).strip().split('/')
    self.sysinfo["procRunning"] = procs[0].strip()
    self.sysinfo["procTotal"] = procs[1].strip()


  def meminfo(self):
    fields = ["MemTotal", "MemFree", "SwapTotal", "SwapFree"]

    (retval, meminfo, err) = run_cmd("cat /proc/meminfo")
    for line in meminfo.split('\n'):
      if line.strip() != "":
        (name, value) = line.split(':')
        name.strip()
        value.strip()
        if name in fields:
          self.sysinfo[name[0].lower() + name[1:]] = value.split()[0]


  def osinfo(self):
    self.sysinfo["nodeName"] = self.node_name
    if not self.is_windows():
      attr = {"osName": "s", "release": "r", "version":"v", "machine":"i"}
      for a in attr.keys():
        (ret, temp, err) = run_cmd("uname -%s" % attr[a])
        self.sysinfo[a] = temp.strip()
      (retval, temp, err) = run_cmd("cat /etc/redhat-release")
      self.sysinfo["distro"] = temp.strip()


  def update_metadata(self):
    log(logging.DEBUG, self.log_name, "Updating system metadata")
    self.loadavg()
    self.meminfo()
    self.osinfo()
    path = "/meta/sysinfo/%s" % self.node_name
    conn = wallaroo_config.ConnectionMeta(**self.connect_opts)
    meta = conn.fetch_json_resource(path, False, default={})
    meta.update(self.sysinfo)
    conn.put_json_resource(path, meta, False)


  def main(self):
    try:
      self.init()

      log(logging.INFO, self.log_name, "Starting Up")
      log(logging.INFO, self.log_name, "Hostname is \"%s\"" % self.node_name)

      self.config()

      # Remove all temp configuration files
      log(logging.INFO, self.log_name, "Cleaning up temporary configuration files")
      temp_dir = tempfile.gettempdir()
      for fname in os.listdir(temp_dir):
        full_path = os.path.normpath("%s/%s" % (temp_dir, fname))
        if self.old_config_filename in fname:
          log(logging.DEBUG, self.log_name, "Deleting temporary configuration file %s" % full_path)
          try:
            os.remove(full_path)
          except Exception, e:
            log(logging.WARNING, self.log_name, e)

      if self.retrieve == True:
        if self.check_config_ver(force=True) == False:
          print "Error: Unable to retrieve configuration"
          log(logging.CRITICAL, self.log_name, "Unable to retrieve configuration")
          self.shutdown()
          return(1)
        self.shutdown()
        return(0)

      # Delay initial checkin after startup 
      random.seed()
      time.sleep(random.randint(0, 10))
      self.check_config_ver()
      self.setup_timers()

      # Loop forever until told to shutdown
      while self.stop_running == False or self.replacing_file == True:
        try:
          time.sleep(1)
          if self.notifications == True:
            self.check_for_notification()

               # Verify timers are still running
          self.monitor_timers()
        except Exception, e:
          log(logging.WARNING, self.log_name, e)
          pass

      self.shutdown()
      return(0)
    except Exception, e:
      if self.stop_running == True:
        self.shutdown()
        return(0)
      else:
        log(logging.ERROR, self.log_name, e)
        self.shutdown()
        return(1)


if __name__ == "__main__":
   sys.exit(Configd().main())