Commit 30726141 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis
Browse files

burnin: Parse arguments

parent 0968f9ca
...@@ -58,11 +58,9 @@ CLASSIFIERS = [] ...@@ -58,11 +58,9 @@ CLASSIFIERS = []
# Package requirements # Package requirements
INSTALL_REQUIRES = [ INSTALL_REQUIRES = [
"IPy", "IPy",
"unittest2",
"python-prctl",
"paramiko", "paramiko",
"vncauthproxy", "vncauthproxy",
"kamaki >= 0.9"] "kamaki >= 0.10"]
setup( setup(
name='snf-tools', name='snf-tools',
......
# Copyright 2013 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""
Burnin: functional tests for Synnefo
"""
import sys
import optparse
from synnefo_tools import version
from synnefo_tools.burnin import common
from synnefo_tools.burnin.astakos_tests import AstakosTestCase, AstakosFoo
# --------------------------------------------------------------------
# Define our TestSuites
TESTSUITES = [
AstakosTestCase, AstakosFoo
]
TSUITES_NAMES = [tsuite.__name__ for tsuite in TESTSUITES]
def string_to_class(names):
"""Convert class namesto class objects"""
return [eval(name) for name in names]
# --------------------------------------------------------------------
# Parse arguments
def parse_comma(option, _, value, parser):
"""Parse comma separated arguments"""
tests = set(TSUITES_NAMES)
parse_input = value.split(',')
if not (set(parse_input)).issubset(tests):
raise optparse.OptionValueError("The selected set of tests is invalid")
setattr(parser.values, option.dest, value.split(','))
def parse_arguments(args):
"""Parse burnin arguments"""
kwargs = {}
kwargs["usage"] = "%prog [options]"
kwargs["description"] = \
"%prog runs a number of test scenarios on a Synnefo deployment."
# Used * or ** magic. pylint: disable-msg=W0142
parser = optparse.OptionParser(**kwargs)
parser.disable_interspersed_args()
parser.add_option(
"--auth-url", action="store",
type="string", default=None, dest="auth_url",
help="The AUTH URI to use to reach the Synnefo API")
parser.add_option(
"--token", action="store",
type="string", default=None, dest="token",
help="The token to use for authentication to the API")
parser.add_option(
"--failfast", action="store_true",
default=False, dest="failfast",
help="Fail immediately if one of the tests fails")
parser.add_option(
"--no-ipv6", action="store_false",
default=True, dest="use_ipv6",
help="Disable IPv6 related tests")
parser.add_option(
"--action-timeout", action="store",
type="int", default=300, dest="action_timeout", metavar="TIMEOUT",
help="Wait TIMEOUT seconds for a server action to complete, "
"then the test is considered failed")
parser.add_option(
"--action-warning", action="store",
type="int", default=120, dest="action_warning", metavar="TIMEOUT",
help="Warn if TIMEOUT seconds have passed and a server action "
"has not been completed yet")
parser.add_option(
"--query-interval", action="store",
type="int", default=3, dest="query_interval", metavar="INTERVAL",
help="Query server status when requests are pending "
"every INTERVAL seconds")
parser.add_option(
"--force-flavor", action="store",
type="string", default=None, dest="force_flavor", metavar="FLAVOR",
help="Force all server creations to use the specified FLAVOR "
"instead of a randomly chosen one. Supports both search by name "
"(reg expression) with \"name:flavor name\" or by id with "
"\"id:flavor id\"")
parser.add_option(
"--force-image", action="store",
type="string", default=None, dest="force_image", metavar="IMAGE",
help="Force all server creations to use the specified IMAGE "
"instead of the default one (a Debian Base image). Just like the "
"--force-flavor option, it supports both search by name and id")
parser.add_option(
"--show-stale", action="store_true",
default=False, dest="show_stale",
help="Show stale servers from previous runs. A server is considered "
"stale if its name starts with `%s'. If stale servers are found, "
"exit with exit status 1." % common.SNF_TEST_PREFIX)
parser.add_option(
"--delete-stale", action="store_true",
default=False, dest="delete_stale",
help="Delete stale servers from previous runs")
parser.add_option(
"--log-folder", action="store",
type="string", default="/var/log/burnin/", dest="log_folder",
help="Define the absolute path where the output log is stored")
parser.add_option(
"--verbose", "-v", action="store",
type="int", default=1, dest="verbose",
help="Print detailed output messages")
parser.add_option(
"--version", action="store_true",
default=False, dest="show_version",
help="Show version and exit")
parser.add_option(
"--set-tests", action="callback", callback=parse_comma,
type="string", default="all", dest="tests",
help="Set comma separated tests for this run. Available tests: %s"
% ", ".join(TSUITES_NAMES))
parser.add_option(
"--exclude-tests", action="callback", callback=parse_comma,
type="string", default=None, dest="exclude_tests",
help="Set comma separated tests to be excluded for this run.")
parser.add_option(
"--no-colors", action="store_false",
default=True, dest="use_colors",
help="Disable colorful output")
(opts, args) = parser.parse_args(args)
# ----------------------------------
# Verify arguments
# If `version' is given show version and exit
if opts.show_version:
show_version()
sys.exit(0)
# `token' is mandatory
mandatory_argument(opts.token, "--token")
# `auth_url' is mandatory
mandatory_argument(opts.auth_url, "--auth-url")
return (opts, args)
def show_version():
"""Show burnin's version"""
sys.stdout.write("Burnin: version %s\n" % version.__version__)
def mandatory_argument(value, arg_name):
"""Check if a mandatory argument is given"""
if (value is None) or (value == ""):
sys.stderr.write("The " + arg_name + " argument is mandatory.\n")
sys.exit("Invalid input")
# --------------------------------------------------------------------
# Burnin main function
def main():
"""Assemble test cases into a test suite, and run it
IMPORTANT: Tests have dependencies and have to be run in the specified
order inside a single test case. They communicate through attributes of the
corresponding TestCase class (shared fixtures). Distinct subclasses of
TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
test runner processes.
"""
# Parse arguments using `optparse'
(opts, _) = parse_arguments(sys.argv[1:])
# Initialize burnin
testsuites = common.initialize(opts, TSUITES_NAMES)
testsuites = string_to_class(testsuites)
# Run burnin
# The return value denotes the success status
return common.run(testsuites)
if __name__ == "__main__":
sys.exit(main())
...@@ -39,7 +39,7 @@ This is the burnin class that tests the Astakos functionality ...@@ -39,7 +39,7 @@ This is the burnin class that tests the Astakos functionality
from kamaki.clients.compute import ComputeClient from kamaki.clients.compute import ComputeClient
from kamaki.clients import ClientError from kamaki.clients import ClientError
import common from synnefo_tools.burnin import common
# Too many public methods (47/20). pylint: disable-msg=R0904 # Too many public methods (47/20). pylint: disable-msg=R0904
...@@ -48,8 +48,8 @@ class AstakosTestCase(common.BurninTests): ...@@ -48,8 +48,8 @@ class AstakosTestCase(common.BurninTests):
def test_unauthorized_access(self): def test_unauthorized_access(self):
"""Test access without a valid token fails""" """Test access without a valid token fails"""
false_token = "12345" false_token = "12345"
client = ComputeClient(self.compute_url, false_token) client = ComputeClient(self.clients.compute_url, false_token)
client.CONNECTION_RETRY_LIMIT = self.connection_retry_limit client.CONNECTION_RETRY_LIMIT = self.clients.retry
with self.assertRaises(ClientError) as cl_error: with self.assertRaises(ClientError) as cl_error:
client.list_servers() client.list_servers()
......
...@@ -49,14 +49,13 @@ except ImportError: ...@@ -49,14 +49,13 @@ except ImportError:
from kamaki.clients.astakos import AstakosClient from kamaki.clients.astakos import AstakosClient
from kamaki.clients.compute import ComputeClient from kamaki.clients.compute import ComputeClient
from logger import Log from synnefo_tools.burnin.logger import Log
# -------------------------------------------------------------------- # --------------------------------------------------------------------
# Global variables # Global variables
logger = None # Invalid constant name. pylint: disable-msg=C0103 logger = None # Invalid constant name. pylint: disable-msg=C0103
AUTH_URL = "https://accounts.okeanos.grnet.gr/identity/v2.0/" SNF_TEST_PREFIX = "snf-test-"
TOKEN = ""
CONNECTION_RETRY_LIMIT = 2 CONNECTION_RETRY_LIMIT = 2
...@@ -99,14 +98,29 @@ class BurninTestResult(unittest.TestResult): ...@@ -99,14 +98,29 @@ class BurninTestResult(unittest.TestResult):
# -------------------------------------------------------------------- # --------------------------------------------------------------------
# BurninTests class # BurninTests class
# Too few public methods (0/2). pylint: disable-msg=R0903
class Clients(object):
"""Our kamaki clients"""
auth_url = None
token = None
astakos = None
retry = CONNECTION_RETRY_LIMIT
compute = None
compute_url = None
# Too many public methods (45/20). pylint: disable-msg=R0904 # Too many public methods (45/20). pylint: disable-msg=R0904
class BurninTests(unittest.TestCase): class BurninTests(unittest.TestCase):
"""Common class that all burnin tests should implement""" """Common class that all burnin tests should implement"""
clients = Clients()
opts = None
@classmethod @classmethod
def setUpClass(cls): # noqa def setUpClass(cls): # noqa
"""Initialize BurninTests""" """Initialize BurninTests"""
cls.suite_name = cls.__name__ cls.suite_name = cls.__name__
cls.connection_retry_limit = CONNECTION_RETRY_LIMIT
logger.testsuite_start(cls.suite_name) logger.testsuite_start(cls.suite_name)
# Set test parameters # Set test parameters
...@@ -115,16 +129,17 @@ class BurninTests(unittest.TestCase): ...@@ -115,16 +129,17 @@ class BurninTests(unittest.TestCase):
def test_clients_setup(self): def test_clients_setup(self):
"""Initializing astakos/cyclades/pithos clients""" """Initializing astakos/cyclades/pithos clients"""
# Update class attributes # Update class attributes
cls = type(self) self.info("Astakos auth url is %s", self.clients.auth_url)
self.info("Astakos auth url is %s", AUTH_URL) self.clients.astakos = AstakosClient(
cls.astakos = AstakosClient(AUTH_URL, TOKEN) self.clients.auth_url, self.clients.token)
cls.astakos.CONNECTION_RETRY_LIMIT = CONNECTION_RETRY_LIMIT self.clients.astakos.CONNECTION_RETRY_LIMIT = self.clients.retry
cls.compute_url = \ self.clients.compute_url = \
cls.astakos.get_service_endpoints('compute')['publicURL'] self.clients.astakos.get_service_endpoints('compute')['publicURL']
self.info("Cyclades url is %s", cls.compute_url) self.info("Cyclades url is %s", self.clients.compute_url)
cls.compute = ComputeClient(cls.compute_url, TOKEN) self.clients.compute = ComputeClient(
cls.compute.CONNECTION_RETRY_LIMIT = CONNECTION_RETRY_LIMIT self.clients.compute_url, self.clients.token)
self.clients.compute.CONNECTION_RETRY_LIMIT = self.clients.retry
def log(self, msg, *args): def log(self, msg, *args):
"""Pass the section value to logger""" """Pass the section value to logger"""
...@@ -147,19 +162,63 @@ class BurninTests(unittest.TestCase): ...@@ -147,19 +162,63 @@ class BurninTests(unittest.TestCase):
logger.error(self.suite_name, msg, *args) logger.error(self.suite_name, msg, *args)
# --------------------------------------------------------------------
# Initialize Burnin
def initialize(opts, testsuites):
"""Initalize burnin
Initialize our logger and burnin state
"""
# Initialize logger
global logger # Using global statement. pylint: disable-msg=C0103,W0603
logger = Log(opts.log_folder, verbose=opts.verbose,
use_colors=opts.use_colors, in_parallel=False)
# Initialize clients
Clients.auth_url = opts.auth_url
Clients.token = opts.token
# Pass the rest options to BurninTests
BurninTests.opts = opts
# Choose tests to run
if opts.tests != "all":
testsuites = opts.tests
if opts.exclude_tests is not None:
testsuites = [tsuite for tsuite in testsuites
if tsuite not in opts.exclude_tests]
return testsuites
# --------------------------------------------------------------------
# Run Burnin
def run(testsuites):
"""Run burnin testsuites"""
global logger # Using global. pylint: disable-msg=C0103,W0603,W0602
success = True
for tcase in testsuites:
tsuite = unittest.TestLoader().loadTestsFromTestCase(tcase)
results = tsuite.run(BurninTestResult())
success = success and \
was_successful(tcase.__name__, results.wasSuccessful())
# Clean up our logger
del(logger)
# Return
return 0 if success else 1
# -------------------------------------------------------------------- # --------------------------------------------------------------------
# Helper functions # Helper functions
def was_succesful(tsuite, success): def was_successful(tsuite, success):
"""Handle whether a testsuite was succesful or not""" """Handle whether a testsuite was succesful or not"""
if success: if success:
logger.testsuite_success(tsuite) logger.testsuite_success(tsuite)
return True
else: else:
logger.testsuite_failure(tsuite) logger.testsuite_failure(tsuite)
return False
def setup_logger(output_dir, verbose=1, use_colors=True, in_parallel=False):
"""Setup our logger"""
global logger # Using global statement. pylint: disable-msg=C0103,W0603
logger = Log(output_dir, verbose=verbose,
use_colors=use_colors, in_parallel=in_parallel)
...@@ -54,7 +54,7 @@ import sys ...@@ -54,7 +54,7 @@ import sys
import os.path import os.path
import datetime import datetime
import filelocker from synnefo_tools.burnin import filelocker
# -------------------------------------------------------------------- # --------------------------------------------------------------------
...@@ -87,19 +87,14 @@ def _red(msg): ...@@ -87,19 +87,14 @@ def _red(msg):
return "\x1b[31m" + str(msg) + "\x1b[0m" return "\x1b[31m" + str(msg) + "\x1b[0m"
def _ts_start(msg): def _magenta(msg):
"""New testsuite color""" """Magenta color"""
return "\x1b[35m" + str(msg) + "\x1b[0m" return "\x1b[35m" + str(msg) + "\x1b[0m"
def _ts_success(msg): def _green(msg):
"""Testsuite passed color""" """Green color"""
return "\x1b[42m" + str(msg) + "\x1b[0m" return "\x1b[32m" + str(msg) + "\x1b[0m"
def _ts_failure(msg):
"""Testsuite failed color"""
return "\x1b[41m" + str(msg) + "\x1b[0m"
def _format_message(msg, *args): def _format_message(msg, *args):
...@@ -155,7 +150,7 @@ def _locate_input(contents, section): ...@@ -155,7 +150,7 @@ def _locate_input(contents, section):
# We didn't find our section?? # We didn't find our section??
sys.stderr.write("Section %s could not be found in logging file\n" sys.stderr.write("Section %s could not be found in logging file\n"
% section) % section)
sys.exit(1) sys.exit("Error in logger._locate_input")
def _add_testsuite_results(contents, section, testsuite): def _add_testsuite_results(contents, section, testsuite):
...@@ -176,7 +171,7 @@ def _add_testsuite_results(contents, section, testsuite): ...@@ -176,7 +171,7 @@ def _add_testsuite_results(contents, section, testsuite):
else: else:
sys.stderr.write("Unknown section %s in _add_testsuite_results\n" sys.stderr.write("Unknown section %s in _add_testsuite_results\n"
% section) % section)
sys.exit(1) sys.exit("Error in logger._add_testsuite_results")
return contents return contents
...@@ -236,11 +231,20 @@ class Log(object): ...@@ -236,11 +231,20 @@ class Log(object):
self.use_colors = use_colors self.use_colors = use_colors
self.in_parallel = in_parallel self.in_parallel = in_parallel
assert output_dir
# Create file for logging # Create file for logging
output_dir = os.path.expanduser(output_dir) output_dir = os.path.expanduser(output_dir)
if not os.path.exists(output_dir): if not os.path.exists(output_dir):
self.debug(None, "Creating directory %s", output_dir) self.debug(None, "Creating directory %s", output_dir)
os.makedirs(output_dir) try:
os.makedirs(output_dir)
except OSError as err:
msg = ("Failed to create folder \"%s\" with error: %s\n"
% (output_dir, err))
sys.stderr.write(msg)
sys.exit("Failed to create log folder")
timestamp = datetime.datetime.strftime( timestamp = datetime.datetime.strftime(
datetime.datetime.now(), "%Y%m%d%H%M%S (%a %b %d %Y %H:%M)") datetime.datetime.now(), "%Y%m%d%H%M%S (%a %b %d %Y %H:%M)")
file_name = timestamp + ".log" file_name = timestamp + ".log"
...@@ -267,11 +271,12 @@ class Log(object): ...@@ -267,11 +271,12 @@ class Log(object):
def __del__(self): def __del__(self):
"""Delete the Log object""" """Delete the Log object"""
# Remove the lock file # Remove the lock file
file_lock = os.path.splitext(self.file_location)[0] + LOCK_EXT if hasattr(self, "file_location"):
try: file_lock = os.path.splitext(self.file_location)[0] + LOCK_EXT
os.remove(file_lock) try:
except OSError: os.remove(file_lock)
self.debug(None, "Couldn't delete lock file") except OSError:
self.debug(None, "Couldn't delete lock file")
# ---------------------------------- # ----------------------------------
# Logging methods # Logging methods
...@@ -377,7 +382,7 @@ class Log(object): ...@@ -377,7 +382,7 @@ class Log(object):
# Add new section to the stdout # Add new section to the stdout
msg = "Starting testsuite %s" % testsuite msg = "Starting testsuite %s" % testsuite
colored_msg = self._color_message(_ts_start, msg) colored_msg = self._color_message(_magenta, msg)
self._write_to_stdout(None, colored_msg) self._write_to_stdout(None, colored_msg)
def testsuite_success(self, testsuite): def testsuite_success(self, testsuite):
...@@ -393,7 +398,7 @@ class Log(object): ...@@ -393,7 +398,7 @@ class Log(object):
# Add success to stdout # Add success to stdout
msg = "Testsuite %s passed" % testsuite msg = "Testsuite %s passed" % testsuite
colored_msg = self._color_message(_ts_success, msg) colored_msg = self._color_message(_green, msg)
self._write_to_stdout(None, colored_msg) self._write_to_stdout(None, colored_msg)
def testsuite_failure(self, testsuite): def testsuite_failure(self, testsuite):
...@@ -409,7 +414,7 @@ class Log(object): ...@@ -409,7 +414,7 @@ class Log(object):
# Add success to stdout # Add success to stdout
msg = "Testsuite %s failed" % testsuite msg = "Testsuite %s failed" % testsuite
colored_msg = self._color_message(_ts_failure, msg) colored_msg = self._color_message(_red, msg)
self._write_to_stdout(None, colored_msg) self._write_to_stdout(None, colored_msg)
# ---------------------------------- # ----------------------------------
......