Commit 5566c005 authored by Stratos Psomadakis's avatar Stratos Psomadakis
Browse files

Add greenlet tracing code in snf-common

Add greenlet tracing code in snf-common, useful for debugging stuck
gevent gunicorn workers.

If the environmental variable SYNNEFO_TRACE is set to a non-empty
string, the tracing code will install a SIGTRAP signal handler for the
process, after the Synnefo settings have been imported.

Sending a SIGTRAP signal to the process (eg a gevent gunicorn worker)
will then print the stack trace of each greenlet spawned within the
process to stderr.
parent c6174c57
......@@ -6,6 +6,19 @@ Unified Changelog file for Synnefo versions >= 0.13
Since v0.13 most of the Synnefo components have been merged into a single
repository and have aligned versions.
.. _Changelog-0.14.3:
v0.14.3
=======
Synnefo-wide
------------
* Use the SYNNEFO_TRACE environmental variable to control whether the greenlet
tracing code will get loaded or not.
Released: UNRELEASED
.. _Changelog-0.14.2:
v0.14.2
......
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
def set_signal_trap():
from os import getpid
from traceback import format_stack, print_exc
from signal import signal, SIGTRAP
from sys import stderr
import gc
def greenlet_trace(arg):
i = 0
stderr.write("--- Greenlet trace: %s\n" % arg)
for ob in gc.get_objects():
if not isinstance(ob, greenlet):
continue
if not ob:
continue
i = i + 1
stderr.write(("--- > Greenlet %d:\n" % i +
"".join(format_stack(ob.gr_frame)) + "\n\n"))
stderr.write("--- End of trace: %s\n" % arg)
try:
from greenlet import greenlet
except ImportError:
def greenlet_trace(arg):
return
def handle_trap(*args):
try:
import trap_inject
reload(trap_inject)
trap_inject.inject()
except ImportError:
pass
except:
print_exc()
msg = ('=== pid: %s' % getpid()) + '\n'.join(format_stack()) + '\n'
stderr.write(msg)
greenlet_trace('TRAP')
signal(SIGTRAP, handle_trap)
......@@ -57,14 +57,23 @@ if os.path.exists(SYNNEFO_SETTINGS_DIR):
conffiles = [f for f in entries if os.path.isfile(f) and
f.endswith(".conf")]
except Exception as e:
print >>sys.stderr, "Failed to list *.conf files under %s" % \
SYNNEFO_SETTINGS_DIR
print >> sys.stderr, "Failed to list *.conf files under %s" % \
SYNNEFO_SETTINGS_DIR
raise SystemExit(1)
conffiles.sort()
for f in conffiles:
try:
execfile(os.path.abspath(f))
except Exception as e:
print >>sys.stderr, "Failed to read settings file: %s [%r]" % \
(os.path.abspath(f), e)
print >> sys.stderr, "Failed to read settings file: %s [%r]" % \
(os.path.abspath(f), e)
raise SystemExit(1)
from os import environ
# The tracing code is enabled by an environmental variable and not a synnefo
# setting, on purpose, so that you can easily control whether it'll get loaded
# or not, based on context (eg enable it for gunicorn but not for eventd).
if environ.get('SYNNEFO_TRACE'):
from synnefo.lib import trace
trace.set_signal_trap()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment