Commit 02d53d23 authored by Vangelis Koukis's avatar Vangelis Koukis
Browse files

Improve logging in logic/dispatcher.py

 * Include file-based logging in /var/log/synnefo/dispatcher.log
   for the dispatcher in the default configuration.
 * Have the dispatcher redirect stdout and stderr file descriptors
   to the first file-based logfile.
 * Add a few log entries wrt dropped AMQP connections
parent 7712ce50
......@@ -4,6 +4,12 @@ README.Upgrade
This file documents the upgrade to newer versions of the Synnefo software.
For more information, please see README.deploy.
v0.5.4 -> v0.5.5
LOGGING
* Changed the default logging settings for the dispatcher to also log
to /var/log/synnefo/dispatcher.log, redirecting stderr and stdout there
v0.5.3 -> v0.5.4
REPOSITORY
* Split Synnefo Ganeti tools to snf-ganeti-tools, with Debian packaging
......
......@@ -54,6 +54,8 @@ import time
import socket
from daemon import daemon
import traceback
# Take care of differences between python-daemon versions.
try:
from daemon import pidfile
......@@ -76,7 +78,7 @@ class Dispatcher:
clienttags = []
def __init__(self, debug = False):
# Initialize logger
self.logger = log.get_logger('synnefo.dispatcher')
......@@ -117,7 +119,9 @@ class Dispatcher:
password=settings.RABBIT_PASSWORD,
virtual_host=settings.RABBIT_VHOST)
except socket.error:
time.sleep(1)
self.logger.error("Failed to connect to %s, retrying in 10s...",
settings.RABBIT_HOST)
time.sleep(10)
self.logger.info("Connection succesful, opening channel")
self.chan = conn.channel()
......@@ -197,15 +201,17 @@ def _init_queues():
def _exit_handler(signum, frame):
""""Catch exit signal in children processes."""
print "%d: Caught signal %d, will raise SystemExit" % (os.getpid(), signum)
""""Catch exit signal in children processes"""
global logger
logger.info("Caught signal %d, will raise SystemExit", signum)
raise SystemExit
def _parent_handler(signum, frame):
""""Catch exit signal in parent process and forward it to children."""
global children
print "Caught signal %d, sending kill signal to children" % signum
global children, logger
logger.info("Caught signal %d, sending SIGTERM to children %s",
signum, children)
[os.kill(pid, SIGTERM) for pid in children]
......@@ -214,7 +220,7 @@ def child(cmdline):
# Cmd line argument parsing
(opts, args) = parse_arguments(cmdline)
disp = Dispatcher(debug = opts.debug)
disp = Dispatcher(debug=opts.debug)
# Start the event loop
disp.wait()
......@@ -245,7 +251,7 @@ def parse_arguments(args):
return parser.parse_args(args)
def purge_queues() :
def purge_queues():
"""
Delete declared queues from RabbitMQ. Use with care!
"""
......@@ -363,44 +369,11 @@ def debug_mode():
disp.wait()
def main():
def daemon_mode(opts):
global children, logger
(opts, args) = parse_arguments(sys.argv[1:])
logger = log.get_logger("synnefo.dispatcher")
# Init the global variables containing the queues
_init_queues()
# Special case for the clean up queues action
if opts.purge_queues:
purge_queues()
return
# Special case for the clean up exch action
if opts.purge_exchanges:
purge_exchanges()
return
if opts.drain_queue:
drain_queue(opts.drain_queue)
return
# Debug mode, process messages without spawning workers
if opts.debug:
log.console_output(logger)
debug_mode()
return
# Become a daemon
daemon_context = daemon.DaemonContext(
stdout=sys.stdout,
stderr=sys.stderr,
umask=022)
daemon_context.open()
# Create pidfile. Take care of differences between python-daemon versions.
# Create pidfile,
# take care of differences between python-daemon versions
try:
pidf = pidfile.TimeoutPIDLockFile(opts.pid_file, 10)
except:
......@@ -418,7 +391,7 @@ def main():
newpid = os.fork()
if newpid == 0:
signal(SIGINT, _exit_handler)
signal(SIGINT, _exit_handler)
signal(SIGTERM, _exit_handler)
child(sys.argv[1:])
sys.exit(1)
......@@ -429,11 +402,11 @@ def main():
i += 1
# Catch signals to ensure graceful shutdown
signal(SIGINT, _parent_handler)
signal(SIGINT, _parent_handler)
signal(SIGTERM, _parent_handler)
# Wait for all children processes to die, one by one
try :
try:
for pid in children:
try:
os.waitpid(pid, 0)
......@@ -442,6 +415,63 @@ def main():
finally:
pidf.release()
def main():
global logger
(opts, args) = parse_arguments(sys.argv[1:])
logger = log.get_logger("synnefo.dispatcher")
# Init the global variables containing the queues
_init_queues()
# Special case for the clean up queues action
if opts.purge_queues:
purge_queues()
return
# Special case for the clean up exch action
if opts.purge_exchanges:
purge_exchanges()
return
if opts.drain_queue:
drain_queue(opts.drain_queue)
return
# Debug mode, process messages without spawning workers
if opts.debug:
log.console_output(logger)
debug_mode()
return
# Redirect stdout and stderr to the fileno of the first
# file-based handler for this logger
stdout_stderr_handler = None
files_preserve = None
for handler in logger.handlers:
if hasattr(handler, 'stream') and hasattr(handler.stream, 'fileno'):
stdout_stderr_handler = handler.stream
files_preserve = [handler.stream]
break
daemon_context = daemon.DaemonContext(
stdout=stdout_stderr_handler,
stderr=stdout_stderr_handler,
files_preserve=files_preserve,
umask=022)
daemon_context.open()
# Catch every exception, make sure it gets logged properly
try:
daemon_mode(opts)
except Exception:
exc = "".join(traceback.format_exception(*sys.exc_info()))
logger.critical(exc)
raise
if __name__ == "__main__":
sys.exit(main())
......
......@@ -2,7 +2,7 @@
keys=root,synnefo.dispatcher,synnefo.logic,synnefo.api,synnefo.invitations,amqplib,rapi
[handlers]
keys=consoleHandler,syslogHandler
keys=consoleHandler,syslogHandler,fileHandler
[formatters]
keys=fullFormatter
......@@ -16,7 +16,7 @@ handlers=syslogHandler
[logger_synnefo.dispatcher]
level=DEBUG
#handlers=consoleHandler
handlers=syslogHandler
handlers=syslogHandler,fileHandler
qualname=synnefo.dispatcher
propagate=0
......@@ -73,6 +73,12 @@ args=("/dev/log",handlers.logging.handlers.SysLogHandler.LOG_DAEMON,)
# The Mac thinks different...
#args=('/var/run/syslog',handlers.SysLogHandler.LOG_DAEMON,)
[handler_fileHandler]
class=logging.FileHandler
level=DEBUG
formatter=fullFormatter
args=('/var/log/synnefo/dispatcher.log',)
# Formatters
[formatter_fullFormatter]
format=%(asctime)s - %(name)s[%(process)d] - %(levelname)s - %(message)s
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment