Skip to content

GitLab

Commit 8a8335cd authored by Kostas Papadimitriou's avatar Kostas Papadimitriou
Browse files

Merge branch 'master' into ui-refactor

parents 43df6199 03fd5271
Hide whitespace changes
Inline Side-by-side
README.admin
View file @ 8a8335cd
......@@ -61,13 +61,18 @@ and is broken in 4 separate dictionaries:
* DISPATCHER_LOGGING is the logging configuration of the logic/dispatcher.py
command line tool.
* RECONCILIATION_LOGGING is the logging configuration of the
logic/reconciliation.py command line tool.
* SNFADMIN_LOGGING is the logging configuration of the snf-admin tool.
Consider using matching configuration for snf-admin and the synnefo.admin
logger of the web app.
Please note the following:
* As of Synnefo v0.7, by default the Django webapp logs to syslog, the
dispatcher logs to /var/log/synnefo/dispatcher.log and the console,
snf-admin logs to the console.
* Different handlers can be set to different logging levels:
for example, everything may appear to the console, but only INFO and higher
may actually be stored in a longer-term logfile.
Admin Tools
===========
......
README.upgrade
View file @ 8a8335cd
......@@ -18,6 +18,8 @@ NEW DEPENDENCIES
Step 13.
COMPONENTS
* snf-admin has been updated with new functionality, be sure to upgrade any
locally installed versions.
* snf-image replaces snf-ganeti-instance-image as the Ganeti OS provider
used by Synnefo, and can live alongside snf-ganeti-instance-image.
Once snf-image has been deployed on all Ganeti nodes, be sure to modify
......@@ -74,8 +76,9 @@ DB MIGRATION
A database migration is needed.
LOGGING
* A new logging mechanism has been implemeted. See 00-logging.conf in
settings.
* A new logging mechanism has been implemeted. Please see 00-logging.conf
under settings.d/ and read the relevant section in README.admin for more
info.
v0.6.1 -> v0.6.2
......
aai/tests.py
View file @ 8a8335cd
......@@ -41,7 +41,7 @@ from synnefo.aai.shibboleth import Tokens
class AaiTestCase(TestCase):
fixtures = ['api_test_data', 'auth_test_data']
fixtures = ['users', 'api_test_data', 'auth_test_data']
apibase = '/api/v1.1'
def setUp(self):
......
api/tests.py
View file @ 8a8335cd
......@@ -54,7 +54,7 @@ class AaiClient(Client):
class APITestCase(TestCase):
fixtures = ['api_test_data', 'users.json']
fixtures = ['users', 'api_test_data']
test_server_id = 1001
test_image_id = 1
test_flavor_id = 1
......@@ -417,7 +417,7 @@ class BaseTestCase(TestCase):
SERVERS = 1
SERVER_METADATA = 0
IMAGE_METADATA = 0
fixtures = ['users.json']
fixtures = ['users']
def setUp(self):
self.client = AaiClient()
......@@ -819,7 +819,7 @@ class ServerVNCConsole(BaseTestCase):
class AaiTestCase(TestCase):
fixtures = ['api_test_data', 'auth_test_data']
fixtures = ['users', 'api_test_data', 'auth_test_data']
apibase = '/api/v1.1'
def setUp(self):
......
invitations/tests.py
View file @ 8a8335cd
......@@ -37,7 +37,7 @@ from django.conf import settings
class InvitationsTestCase(TestCase):
fixtures = ['users']
token = '46e427d657b20defe352804f0eb6f8a2'
def setUp(self):
......
logic/reconciliation.py
View file @ 8a8335cd
......@@ -132,10 +132,10 @@ def get_instances_from_ganeti():
return snf_instances
# Only for testing this module individually
def main():
print get_instances_from_ganeti()
if __name__ == "__main__":
dictConfig(settings.RECONCILIATION_LOGGING)
sys.exit(main())
runtests.sh 0 → 100755
View file @ 8a8335cd
#!/bin/bash
#
#
# Copyright 2011 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
#
set -e
echo "Running Django tests..." >&2
./manage.py test aai admin api db helpdesk invitations logic
echo $?
echo "Running snf-ganeti-tools tests..." >&2
PYTHONPATH=snf-ganeti-tools:$PYTHONPATH ./snf-ganeti-tools/test/synnefo.ganeti_unittest.py
settings.d/00-logging.conf
View file @ 8a8335cd
......@@ -32,7 +32,7 @@ LOGGING = {
'class': 'logging.handlers.SysLogHandler',
'address': '/dev/log',
# 'address': ('localhost', 514),
'facility': 'logging.handlers.SysLogHandler.LOG_DAEMON',
'facility': 'daemon',
'formatter': 'verbose',
'level': 'INFO',
},
......@@ -40,28 +40,29 @@ LOGGING = {
'loggers': {
'synnefo': {
'handlers': ['console'],
'level': 'DEBUG'
'handlers': ['syslog'],
'level': 'INFO'
},
'synnefo.admin': {
'level': 'DEBUG',
'level': 'INFO',
'propagate': 1
},
'synnefo.api': {
'level': 'DEBUG',
'level': 'INFO',
'propagate': 1
},
'synnefo.db': {
'level': 'DEBUG',
'level': 'INFO',
'propagate': 1
},
'synnefo.logic': {
'level': 'DEBUG',
'level': 'INFO',
'propagate': 1
},
}
}
DISPATCHER_LOGGING = {
'version': 1,
'disable_existing_loggers': True,
......@@ -79,9 +80,9 @@ DISPATCHER_LOGGING = {
},
'file': {
'class': 'logging.handlers.WatchedFileHandler',
'filename': 'dispatcher.log',
'filename': '/var/log/synnefo/dispatcher.log',
'formatter': 'verbose',
'level': 'INFO'
'level': 'DEBUG'
},
},
......@@ -95,32 +96,6 @@ DISPATCHER_LOGGING = {
}
}
RECONCILIATION_LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'verbose': {
'format': '%(asctime)s [%(levelname)s] %(message)s'
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'verbose'
},
},
'loggers': {
'synnefo': {'propagate': 1}
},
'root': {
'handlers': ['console'],
'level': 'DEBUG',
}
}
SNFADMIN_LOGGING = {
'version': 1,
......
snf-ganeti-tools/conf/default/snf-ganeti-eventd 0 → 100644
View file @ 8a8335cd
#!/bin/sh
SNF_EVENTD_ENABLE=false
SNF_USER="root"
SNF_EVENTD_OPTS=""
snf-ganeti-tools/conf/init.d/snf-ganeti-eventd 0 → 100755
View file @ 8a8335cd
#! /bin/sh
### BEGIN INIT INFO
# Provides: snf-ganeti-eventd
# Required-Start: $remote_fs $syslog ganeti
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# X-Start-After: ganeti
# Short-Description: Synnefo ganeti-eventd daemon
# Description: ganeti-eventd is a daemon
### END INIT INFO
set -e
# /etc/init.d/snf-ganeti-eventd: start and stop the ganeti-eventd daemon
# script skeleton stolen from rsyncd
DAEMON=/usr/sbin/snf-ganeti-eventd
SNF_EVENTD_PID_FILE=/var/run/snf-ganeti-eventd.pid
SNF_EVENTD_DEFAULTS=/etc/default/snf-ganeti-eventd
SNF_EVENTD_OPTS=''
SNF_EVENTD_ENABLE=true
SNF_USER="root"
test -x $DAEMON || exit 0
. /lib/lsb/init-functions
if [ -s $SNF_EVENTD_DEFAULTS_FILE ]; then
. $SNF_EVENTD_DEFAULTS_FILE
fi
export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
eventd_start() {
if start-stop-daemon --start --chuid $SNF_USER --pidfile $SNF_EVENTD_PID_FILE \
--exec $DAEMON -- $SNF_EVENTD_OPTS
then
rc=0
sleep 1
if ! kill -0 $(cat $SNF_EVENTD_PID_FILE) >/dev/null 2>&1; then
log_failure_msg "snf-ganeti-eventd daemon failed to start"
rc=1
fi
else
rc=1
fi
if [ $rc -eq 0 ]; then
log_end_msg 0
else
log_end_msg 1
rm -f $SNF_EVENTD_PID_FILE
fi
} # eventd_start
MASTER=`/usr/sbin/gnt-cluster getmaster`
HOST=`/bin/hostname -f`
if [ "x$MASTER" != x$HOST ]] ;
then
log_warning_msg "snf-ganeti-eventd should run on the ganeti master only, aborting"
log_end_msg 0
exit 0
fi
case "$1" in
start)
if "$SNF_EVENTD_ENABLE"; then
log_daemon_msg "Starting snf-ganeti-eventd daemon" "snf-ganeti-eventd"
if [ -s $SNF_EVENTD_PID_FILE ] && kill -0 $(cat $SNF_EVENTD_PID_FILE) >/dev/null 2>&1; then
log_progress_msg "apparently already running"
log_end_msg 0
exit 0
fi
eventd_start
else
if [ -s "$SNF_EVENTD_CONFIG_FILE" ]; then
[ "$VERBOSE" != no ] && log_warning_msg "snf-ganeti-eventd daemon not enabled in $SNF_EVENTD_DEFAULTS_FILE, not starting..."
fi
fi
;;
stop)
log_daemon_msg "Stopping snf-ganeti-eventd daemon" "snf-ganeti-eventd"
start-stop-daemon --stop --quiet --oknodo --pidfile $SNF_EVENTD_PID_FILE
log_end_msg $?
rm -f $SNF_EVENTD_PID_FILE
;;
restart)
set +e
if $SNF_EVENTD_ENABLE; then
log_daemon_msg "Restarting snf-ganeti-eventd daemon" "snf-ganeti-eventd"
if [ -s $SNF_EVENTD_PID_FILE ] && kill -0 $(cat $SNF_EVENTD_PID_FILE) >/dev/null 2>&1; then
start-stop-daemon --stop --quiet --oknodo --pidfile $SNF_EVENTD_PID_FILE || true
sleep 1
else
log_warning_msg "snf-ganeti-eventd daemon not running, attempting to start."
rm -f $SNF_EVENTD_PID_FILE
fi
eventd_start
else
if [ -s "$SNF_EVENTD_CONFIG_FILE" ]; then
[ "$VERBOSE" != no ] && log_warning_msg "snf-ganeti-eventd daemon not enabled in $SNF_EVENTD_DEFAULTS_FILE, not starting..."
fi
fi
;;
status)
status_of_proc -p $SNF_EVENTD_PID_FILE "$DAEMON" ganeti-eventd
exit $? # notreached due to set -e
;;
*)
echo "Usage: /etc/init.d/snf-ganeti-eventd {start|stop|restart|status}"
exit 1
esac
exit 0
snf-ganeti-tools/debian/changelog
View file @ 8a8335cd
snf-ganeti-tools (0.7) UNRELEASED; urgency=low
* New upstream version.
-- Vangelis Koukis <vkoukis@grnet.gr> Tue, 11 Oct 2011 23:01:46 +0300
snf-ganeti-tools (0.6) UNRELEASED; urgency=low
* New upstream version.
......
snf-ganeti-tools/debian/install
View file @ 8a8335cd
kvm-vif-bridge /etc/ganeti
conf/default/snf-ganeti-eventd /etc/default
conf/init.d/snf-ganeti-eventd /etc/init.d
snf-ganeti-tools/debian/postinst 0 → 100644
View file @ 8a8335cd
#!/bin/sh
# postinst script for snf-image-host
#
# see: dh_installdeb(1)
set -e
# summary of how this script can be called:
# * <postinst> `configure' <most-recently-configured-version>
# * <old-postinst> `abort-upgrade' <new version>
# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
# <new-version>
# * <postinst> `abort-remove'
# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
# <failed-install-package> <version> `removing'
# <conflicting-package> <version>
# for details, see http://www.debian.org/doc/debian-policy/ or
# the debian-policy package
case "$1" in
configure)
echo "Updating rc.d links... "
update-rc.d snf-ganeti-eventd defaults
echo "Starting snf-ganeti-eventd...\n"
/etc/init.d/snf-ganeti-eventd start
;;
abort-upgrade|abort-remove|abort-deconfigure)
;;
*)
echo "postinst called with unknown argument \`$1'" >&2
exit 1
;;
esac
# dh_installdeb will replace this with shell code automatically
# generated by other debhelper scripts.
#DEBHELPER#
exit 0
snf-ganeti-tools/debian/prerm 0 → 100644
View file @ 8a8335cd
#!/bin/sh
# prerm script for snf-image-host
#
# see: dh_installdeb(1)
set -e
# summary of how this script can be called:
# * <prerm> `remove'
# * <old-prerm> `upgrade' <new-version>
# * <new-prerm> `failed-upgrade' <old-version>
# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
# * <deconfigured's-prerm> `deconfigure' `in-favour'
# <package-being-installed> <version> `removing'
# <conflicting-package> <version>
# for details, see http://www.debian.org/doc/debian-policy/ or
# the debian-policy package
case "$1" in
remove|upgrade|deconfigure)
echo "Stopping snf-ganeti-eventd..."
/etc/init.d/snf-ganeti-eventd stop
echo "Removing rc.d links... "
update-rc.d snf-ganeti-eventd remove
;;
failed-upgrade)
;;
*)
echo "prerm called with unknown argument \`$1'" >&2
exit 1
;;
esac
# dh_installdeb will replace this with shell code automatically
# generated by other debhelper scripts.
#DEBHELPER#
exit 0
snf-ganeti-tools/setup.py
View file @ 8a8335cd
......@@ -4,7 +4,7 @@ from setuptools import setup
setup(
name="snf-ganeti-tools",
version="0.6",
version="0.7",
description="Synnefo Ganeti supplementary tools",
author="Synnefo Development Team",
author_email="synnefo@lists.grnet.gr",
......
snf-tools/test_suite.py → snf-tools/burnin.py
View file @ 8a8335cd
......@@ -76,7 +76,7 @@ SNF_TEST_PREFIX = "snf-test-"
# Setup logging (FIXME - verigak)
logging.basicConfig(format="%(message)s")
log = logging.getLogger("snf-test")
log = logging.getLogger("burnin")
log.setLevel(logging.INFO)
......@@ -233,7 +233,6 @@ class SpawnServerTestCase(unittest.TestCase):
def _verify_server_status(self, current_status, new_status):
"""Verify a server has switched to a specified status"""
server = self.client.get_server_details(self.serverid)
self.assertIn(server["status"], (current_status, new_status))
if server["status"] not in (current_status, new_status):
return None # Do not raise exception, return so the test fails
self.assertEquals(server["status"], new_status)
......@@ -584,16 +583,14 @@ def _run_cases_in_parallel(cases, fanout=1, runner=None):
The cases iterable specifies the TestCases to be executed in parallel,
by test runners running in distinct processes.
The fanout parameter specifies the number of processes to spawn,
and defaults to 1.
The runner argument specifies the test runner class to use inside each
runner process.
"""
if runner is None:
runner = unittest.TextTestRunner()
runner = unittest.TextTestRunner(verbosity=2, failfast=True)
# testq: The master process enqueues TestCase objects into this queue,
# test runner processes pick them up for execution, in parallel.
......@@ -630,7 +627,10 @@ def _spawn_server_test_case(**kwargs):
inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
if hasattr(m, __doc__):
m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
setattr(__main__,name,cls)
# Make sure the class can be pickled, by listing it among
# the attributes of __main__. A PicklingError is raised otherwise.
setattr(__main__, name, cls)
return cls
......@@ -674,9 +674,10 @@ def parse_arguments(args):
action="store", type="string", dest="token",
help="The token to use for authentication to the API",
default=DEFAULT_TOKEN)
parser.add_option("--failfast",
action="store_true", dest="failfast",
help="Fail immediately if one of the tests fails",
parser.add_option("--nofailfast",
action="store_true", dest="nofailfast",
help="Do not fail immediately if one of the tests " \
"fails (EXPERIMENTAL)",
default=False)
parser.add_option("--action-timeout",
action="store", type="int", dest="action_timeout",
......@@ -707,7 +708,7 @@ def parse_arguments(args):
metavar="COUNT",
help="Spawn up to COUNT child processes to execute " \
"in parallel, essentially have up to COUNT " \
"server build requests outstanding",
"server build requests outstanding (EXPERIMENTAL)",
default=1)
parser.add_option("--force-flavor",
action="store", type="int", dest="force_flavorid",
......@@ -715,7 +716,13 @@ def parse_arguments(args):
help="Force all server creations to use the specified "\
"FLAVOR ID instead of a randomly chosen one, " \
"useful if disk space is scarce",
default=None) # FIXME
default=None)
parser.add_option("--image-id",
action="store", type="string", dest="force_imageid",
metavar="IMAGE ID",
help="Test the specified image id, use 'all' to test " \
"all available images (mandatory argument)",
default=None)
parser.add_option("--show-stale",
action="store_true", dest="show_stale",
help="Show stale servers from previous runs, whose "\
......@@ -736,6 +743,20 @@ def parse_arguments(args):
if opts.delete_stale:
opts.show_stale = True
if not opts.show_stale:
if not opts.force_imageid:
print >>sys.stderr, "The --image-id argument is mandatory."
parser.print_help()
sys.exit(1)
if opts.force_imageid != 'all':
try:
opts.force_imageid = int(opts.force_imageid)
except ValueError:
print >>sys.stderr, "Invalid value specified for --image-id." \
"Use a numeric id, or `all'."
sys.exit(1)
return (opts, args)
......@@ -770,7 +791,7 @@ def main():
# Run them: FIXME: In parallel, FAILEARLY, catchbreak?
#unittest.main(verbosity=2, catchbreak=True)
runner = unittest.TextTestRunner(verbosity=2, failfast=opts.failfast)
runner = unittest.TextTestRunner(verbosity=2, failfast=not opts.nofailfast)
# The following cases run sequentially
seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
_run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
......@@ -778,7 +799,12 @@ def main():
# The following cases run in parallel
par_cases = []
for image in DIMAGES:
if opts.force_imageid == 'all':
test_images = DIMAGES
else:
test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
for image in test_images:
imageid = image["id"]
imagename = image["name"]
if opts.force_flavorid:
......@@ -799,8 +825,6 @@ def main():
query_interval=opts.query_interval)
par_cases.append(case)
print "%s" % FlavorsTestCase
print "dict", __main__.__dict__
_run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
if __name__ == "__main__":
......
snf-tools/snf-admin
View file @ 8a8335cd
......@@ -312,7 +312,7 @@ class RegisterImage(Command):
def add_options(self, parser):
parser.add_option('--meta', dest='meta', action='append',
metavar='KEY=VAL',
help='assign image to user with id UID')
help='add metadata (can be used multiple times)')
parser.add_option('--public', action='store_true', dest='public',
default=False, help='make image public')
parser.add_option('-u', dest='uid', metavar='UID',
......@@ -344,7 +344,7 @@ class RegisterImage(Command):
for m in self.meta:
key, sep, val = m.partition('=')
if key and val:
image.imagemetadata_set.create(meta_key=key, meta_value=val)
image.metadata.create(meta_key=key, meta_value=val)
else:
print 'WARNING: Ignoring meta', m
......
snf-tools/snf-image deleted 120000 → 0
View file @ 43df6199
snf-admin
\ No newline at end of file
snf-tools/snf-server deleted 120000 → 0
View file @ 43df6199
snf-admin
\ No newline at end of file
snf-tools/snf-user deleted 120000 → 0
View file @ 43df6199
snf-admin
\ No newline at end of file