Merge branch 'feature-refactor-burnin' into develop

ci/ci_squeeze.conf

@@ -60,7 +60,11 @@ ssh_keys = ~/.ssh/id_rsa.pub
 [Burnin]
 # Maybe add some burnin options
 # (e.g. tests to run/ignore, timeouts etc)
-cmd_options = --nofailfast --no-ipv6 --action-timeout=240
+cmd_options = --images "name:.*" --flavors "name:C1R512D2file" --no-ipv6
+# Kamaki version to be used (leave empty for default)
+# In some cases there is the need for a specific version
+# of kamaki to be used.
+kamaki_version = 0.11next-1858-1d7368b
 
 
 [Unit Tests]

ci/ci_wheezy.conf

@@ -60,7 +60,11 @@ ssh_keys = ~/.ssh/id_rsa.pub
 [Burnin]
 # Maybe add some burnin options
 # (e.g. tests to run/ignore, timeouts etc)
-cmd_options = --nofailfast --no-ipv6 --action-timeout=240
+cmd_options = --images "name:.*" --flavors "name:C1R512D2file" --no-ipv6
+# Kamaki version to be used (leave empty for default)
+# In some cases there is the need for a specific version
+# of kamaki to be used.
+kamaki_version = 0.11next-1858-1d7368b
 
 
 [Unit Tests]

ci/utils.py

@@ -731,12 +731,13 @@ class SynnefoCI(object):
     def build_packages(self):
         """Build packages needed by Synnefo software"""
         self.logger.info("Install development packages")
+        kamaki_version = self.config.get('Burnin', 'kamaki_version')
         cmd = """
         apt-get update
         apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
                 python-dev python-all python-pip ant --yes --force-yes
-        pip install -U devflow
-        """
+        pip install -U devflow kamaki{0}
+        """.format(("==" + kamaki_version) if kamaki_version else "")
         _run(cmd, False)
 
         # Patch pydist bug
@@ -881,17 +882,8 @@ class SynnefoCI(object):
         token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
         images_user=$(kamaki image list -l | grep owner | \
                       cut -d':' -f2 | tr -d ' ')
-        snf-burnin --auth-url=$auth_url --token=$token \
-            --force-flavor=2 --image-id=all \
-            --system-images-user=$images_user \
-            {0}
+        snf-burnin --auth-url=$auth_url --token=$token {0}
         BurninExitStatus=$?
-        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
-        for i in $(ls $log_folder/*/details*); do
-            echo -e "\\n\\n"
-            echo -e "***** $i\\n"
-            cat $i
-        done
         exit $BurninExitStatus
         """.format(self.config.get('Burnin', 'cmd_options'))
         _run(cmd, True)

snf-deploy/snfdeploy/fabfile.py

@@ -1340,7 +1340,7 @@ def setup_kamaki():
     kamaki config set cloud.default.token {1}
     """.format(env.env.accounts.fqdn, user_auth_token)
     try_run(cmd)
-    try_run("kamaki file create images")
+    try_run("kamaki container create images")
 
 
 @roles("client")

snf-tools/conf/snf-burnin-run.sh

@@ -8,15 +8,15 @@
 # Here we define the tokens for each user burnin will
 # test along with an alias for each token.
 # For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR.
-USERS=(\
-    "burnin1" "token to be used" \
-    "image id to be used" "flavor id to be used" \
+USERS=(
+    "burnin1" "token to be used"
+    "name:Image name (reg expr)" "name:Flavor name (reg expr)"
 
-    "burnin2" "token to be used" \
-    "image id to be used" "flavor id to be used" \
+    "burnin2" "token to be used"
+    "name:Image name (reg expr)" "name:Flavor name (reg expr)"
 
-    "burnin3" "token to be used" \
-    "image id to be used" "flavor id to be used" \
+    "burnin3" "token to be used"
+    "name:Image name (reg expr)" "name:Flavor name (reg expr)"
   )
 
 # ----------------------------------------
@@ -35,16 +35,12 @@ FAILURE_SUBJECT="Burnin Failed"
 # ----------------------------------------
 # Some burnin parameters
 AUTH_URL="https://accounts.synnefo.org/identity/v2.0"
-SYSTEM_IMAGES_USER="uuid-of-owner-of-system-images"
-TIMEOUT=240
 
 # ----------------------------------------
 # Burnin executable and log files
 Burnin="snf-burnin"
 # Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance
-LOGFOLDER="/var/log/burnin_results/"
-# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance
-OUTPUTFOLDER="/tmp"
+LOGFOLDER="/var/log/burnin/"
 # Lock file (we don't want two instances of this script)
 LOCKFILE="/tmp/burnin.lockfile"
 
@@ -60,78 +56,45 @@ run_burnin() {
     local success_subject="$TAG ($alias) $SUCCESS_SUBJECT"
     local failure_subject="$TAG ($alias) $FAILURE_SUBJECT"
     local logfolder="$LOGFOLDER/$alias"
-    local outputfile="$OUTPUTFOLDER/burnin-$alias.out"
-    local failed=false
     local error_summary
     local stale_subject
 
-    # Save date-stamp to output
-    date > $outputfile
-    echo -e \
-        "\n\n===== Burnin Output ========================================" \
-        >> $outputfile
-
     # Check for stale servers/networks
-    $Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \
-        grep "test" >> $outputfile 2>&1
-    if [ $? -ne 0 ]; then
+    $Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale --quiet
+    if [ $? -eq 0 ]; then
         # No stale servers/networks found. Run burnin
-        $Burnin --token="$token" \
-                --action-timeout="$TIMEOUT" \
-                --image-id="$image" \
-                --log-folder="$logfolder" \
-                --auth-url="$AUTH_URL" \
-                --force-flavor="$flavor" \
-                --system-images-user="$SYSTEM_IMAGES_USER" \
-                --nofailfast \
-            &>> $outputfile
-
-        echo -e \
-            "\n\n===== Burnin Logs ==========================================" \
-            >> $outputfile
-
-        # Search log files for errors
-        for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do
-            if egrep "(ERROR)|(FAILED)" $file > /dev/null; then
-                failed=true
-                echo "FILENAME: $file" >> $outputfile
-                echo "ERROR: " >> $outputfile
-                cat "$file" >> $outputfile
-            fi
-        done
-
-        # Clean output file from escape characters
-        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
-
-        # Send emails
-        if $failed; then
-            error_summary="`cat $outputfile | \
-                egrep "ERROR: test_|FAIL: test_" | \
-                awk '{ print $2 }' | grep -v "^$" | \
-                sed -e 's/_/ /g' | cut -d" " -f3- | \
-                tr '\n' ',' | sed -e 's/,$//g' | \
-                sed -e 's/,/,  /g'`"
-            cat $outputfile | /usr/bin/mailx -E \
+        results=$($Burnin \
+            --token="$token" \
+            --auth-url="$AUTH_URL" \
+            --images="$image" \
+            --flavors="$flavor" \
+            --log-folder="$logfolder" \
+            --final-report-only \
+            2>&1)
+
+        if [ $? -ne 0 ]; then
+            # Burnin failed
+            # Send email
+            error_summary=$(echo "$results" | \
+                sed -n 's/  \* Failed: \(.*\)/\1/p')
+            echo "$results" | /usr/bin/mailx -E \
                 -s "$failure_subject: $error_summary" $RECIPIENTS
 #        else
-#            cat $outputfile | /usr/bin/mailx -E \
+#            echo "$results" | /usr/bin/mailx -E \
 #                -s "$success_subject" $RECIPIENTS
         fi
     else
         # Burnin found stale servers/networks. Try to clean them
-        $Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \
-            >> $outputfile 2>&1
+        results=$($Burnin --token="$token" --auth-url="$AUTH_URL" \
+            --delete-stale --log-folder="$logfolder" --final-report-only 2>&1)
         if [ $? -ne 0 ]; then
             stale_subject="$failure_subject: Couldn't delete stale servers/networks"
         else
             stale_subject="$success_subject: Stale servers/networks deleted"
         fi
 
-        # Clean output file from escape characters
-        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
-
         # Send mail
-        cat $outputfile | /usr/bin/mailx -E \
+        echo "$results" | /usr/bin/mailx -E \
             -s "$stale_subject" $RECIPIENTS
     fi
 }
@@ -143,7 +106,7 @@ run_burnin() {
 (
     flock -xn 200 || exit 1
 
-    set ${USERS[@]}
+    set "${USERS[@]}"
 
     while [ -n "$1" ]; do
         run_burnin "$1" "$2" "$3" "$4" &

snf-tools/setup.py

@@ -58,11 +58,9 @@ CLASSIFIERS = []
 # Package requirements
 INSTALL_REQUIRES = [
     "IPy",
-    "unittest2",
-    "python-prctl",
     "paramiko",
     "vncauthproxy",
-    "kamaki >= 0.9"]
+    "kamaki >= 0.10"]
 
 setup(
     name='snf-tools',

snf-tools/synnefo_tools/burnin/__init__.py

+# Copyright 2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+"""
+Burnin: functional tests for Synnefo
+
+"""
+
+import sys
+import optparse
+
+from synnefo_tools import version
+from synnefo_tools.burnin import common
+from synnefo_tools.burnin.astakos_tests import AstakosTestSuite
+from synnefo_tools.burnin.images_tests import \
+    FlavorsTestSuite, ImagesTestSuite
+from synnefo_tools.burnin.pithos_tests import PithosTestSuite
+from synnefo_tools.burnin.server_tests import ServerTestSuite
+from synnefo_tools.burnin.network_tests import NetworkTestSuite
+from synnefo_tools.burnin.stale_tests import \
+    StaleServersTestSuite, StaleFloatingIPsTestSuite, StaleNetworksTestSuite
+
+
+# --------------------------------------------------------------------
+# Define our TestSuites
+TESTSUITES = [
+    AstakosTestSuite,
+    FlavorsTestSuite,
+    ImagesTestSuite,
+    PithosTestSuite,
+    ServerTestSuite,
+    NetworkTestSuite,
+]
+TSUITES_NAMES = [tsuite.__name__ for tsuite in TESTSUITES]
+
+STALE_TESTSUITES = [
+    # Must be runned in this order
+    StaleServersTestSuite,
+    StaleFloatingIPsTestSuite,
+    StaleNetworksTestSuite,
+]
+STALE_TSUITES_NAMES = [tsuite.__name__ for tsuite in STALE_TESTSUITES]
+
+
+def string_to_class(names):
+    """Convert class namesto class objects"""
+    return [eval(name) for name in names]
+
+
+# --------------------------------------------------------------------
+# Parse arguments
+def parse_comma(option, _, value, parser):
+    """Parse comma separated arguments"""
+    parse_input = [p.strip() for p in value.split(',')]
+    setattr(parser.values, option.dest, parse_input)
+
+
+def parse_arguments(args):
+    """Parse burnin arguments"""
+    kwargs = {}
+    kwargs["usage"] = "%prog [options]"
+    kwargs["description"] = \
+        "%prog runs a number of test scenarios on a Synnefo deployment."
+
+    # Used * or ** magic. pylint: disable-msg=W0142
+    parser = optparse.OptionParser(**kwargs)
+    parser.disable_interspersed_args()
+
+    parser.add_option(
+        "--auth-url", action="store",
+        type="string", default=None, dest="auth_url",
+        help="The AUTH URI to use to reach the Synnefo API")
+    parser.add_option(
+        "--token", action="store",
+        type="string", default=None, dest="token",
+        help="The token to use for authentication to the API")
+    parser.add_option(
+        "--failfast", action="store_true",
+        default=False, dest="failfast",
+        help="Fail immediately if one of the tests fails")
+    parser.add_option(
+        "--no-ipv6", action="store_false",
+        default=True, dest="use_ipv6",
+        help="Disable IPv6 related tests")
+    parser.add_option(
+        "--action-timeout", action="store",
+        type="int", default=420, dest="action_timeout", metavar="TIMEOUT",
+        help="Wait TIMEOUT seconds for a server action to complete, "
+             "then the test is considered failed")
+    parser.add_option(
+        "--action-warning", action="store",
+        type="int", default=180, dest="action_warning", metavar="TIMEOUT",
+        help="Warn if TIMEOUT seconds have passed and a server action "
+             "has not been completed yet")
+    parser.add_option(
+        "--query-interval", action="store",
+        type="int", default=3, dest="query_interval", metavar="INTERVAL",
+        help="Query server status when requests are pending "
+             "every INTERVAL seconds")
+    parser.add_option(
+        "--flavors", action="callback", callback=parse_comma,
+        type="string", default=None, dest="flavors", metavar="FLAVORS",
+        help="Force all server creations to use one of the specified FLAVORS "
+             "instead of a randomly chosen one. Supports both search by name "
+             "(reg expression) with \"name:flavor name\" or by id with "
+             "\"id:flavor id\"")
+    parser.add_option(
+        "--images", action="callback", callback=parse_comma,
+        type="string", default=None, dest="images", metavar="IMAGES",
+        help="Force all server creations to use one of the specified IMAGES "
+             "instead of the default one (a Debian Base image). Just like the "
+             "--flavors option, it supports both search by name and id")
+    parser.add_option(
+        "--system-user", action="store",
+        type="string", default=None, dest="system_user",
+        help="Owner of system images (typed option in the form of "
+             "\"name:user_name\" or \"id:uuuid\")")
+    parser.add_option(
+        "--show-stale", action="store_true",
+        default=False, dest="show_stale",
+        help="Show stale servers from previous runs. A server is considered "
+             "stale if its name starts with `%s'. If stale servers are found, "
+             "exit with exit status 1." % common.SNF_TEST_PREFIX)
+    parser.add_option(
+        "--delete-stale", action="store_true",
+        default=False, dest="delete_stale",
+        help="Delete stale servers from previous runs")
+    parser.add_option(
+        "--log-folder", action="store",
+        type="string", default="/var/log/burnin/", dest="log_folder",
+        help="Define the absolute path where the output log is stored")
+    parser.add_option(
+        "--verbose", "-v", action="store",
+        type="int", default=1, dest="verbose",
+        help="Print detailed output messages")
+    parser.add_option(
+        "--version", action="store_true",
+        default=False, dest="show_version",
+        help="Show version and exit")
+    parser.add_option(
+        "--set-tests", action="callback", callback=parse_comma,
+        type="string", default="all", dest="tests",
+        help="Set comma separated tests for this run. Available tests: %s"
+             % ", ".join(TSUITES_NAMES))
+    parser.add_option(
+        "--exclude-tests", action="callback", callback=parse_comma,
+        type="string", default=None, dest="exclude_tests",
+        help="Set comma separated tests to be excluded for this run.")
+    parser.add_option(
+        "--no-colors", action="store_false",
+        default=True, dest="use_colors",
+        help="Disable colorful output")
+    parser.add_option(
+        "--quiet", action="store_true",
+        default=False, dest="quiet",
+        help="Turn off logging (both console and file logging)")
+    parser.add_option(
+        "--final-report-only", action="store_true",
+        default=False, dest="final_report",
+        help="Turn off log output and only print the contents of the log "
+             "file at the end of the test. Useful when burnin is used in "
+             "script files and it's output is to be sent using email")
+
+    (opts, args) = parser.parse_args(args)
+
+    # ----------------------------------
+    # Verify arguments
+    # If `version' is given show version and exit
+    if opts.show_version:
+        show_version()
+        sys.exit(0)
+
+    # `delete_stale' implies `show_stale'
+    if opts.delete_stale:
+        opts.show_stale = True
+
+    # log_level:
+    #  0 -> log to console and file
+    #  1 -> log to file and output the results in console
+    #  2 -> don't log
+    opts.log_level = 0
+    if opts.final_report:
+        opts.log_level = 1
+    if opts.quiet:
+        opts.log_level = 2
+
+    # Check `--set-tests' and `--exclude-tests' options
+    if opts.tests != "all" and \
+            not (set(opts.tests)).issubset(set(TSUITES_NAMES)):
+        raise optparse.OptionValueError("The selected set of tests is invalid")
+    if opts.exclude_tests is not None and \
+            not (set(opts.exclude_tests)).issubset(set(TSUITES_NAMES)):
+        raise optparse.OptionValueError("The selected set of tests is invalid")
+
+    # `token' is mandatory
+    mandatory_argument(opts.token, "--token")
+    # `auth_url' is mandatory
+    mandatory_argument(opts.auth_url, "--auth-url")
+
+    return (opts, args)
+
+
+def show_version():
+    """Show burnin's version"""
+    sys.stdout.write("Burnin: version %s\n" % version.__version__)
+
+
+def mandatory_argument(value, arg_name):
+    """Check if a mandatory argument is given"""
+    if (value is None) or (value == ""):
+        sys.stderr.write("The " + arg_name + " argument is mandatory.\n")
+        sys.exit("Invalid input")
+
+
+# --------------------------------------------------------------------
+# Burnin main function
+def main():
+    """Assemble test cases into a test suite, and run it
+
+    IMPORTANT: Tests have dependencies and have to be run in the specified
+    order inside a single test case. They communicate through attributes of the
+    corresponding TestCase class (shared fixtures). Distinct subclasses of
+    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
+    test runner processes.
+
+    """
+
+    # Parse arguments using `optparse'
+    (opts, _) = parse_arguments(sys.argv[1:])
+
+    # Initialize burnin
+    (testsuites, failfast) = \
+        common.initialize(opts, TSUITES_NAMES, STALE_TSUITES_NAMES)
+    testsuites = string_to_class(testsuites)
+
+    # Run burnin
+    # The return value denotes the success status
+    return common.run_burnin(testsuites, failfast=failfast)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

snf-tools/synnefo_tools/burnin/astakos_tests.py

+# Copyright 2013 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+"""
+This is the burnin class that tests the Astakos functionality
+
+"""
+
+from kamaki.clients.compute import ComputeClient
+from kamaki.clients import ClientError
+
+from synnefo_tools.burnin import common
+
+
+# Too many public methods. pylint: disable-msg=R0904
+class AstakosTestSuite(common.BurninTests):
+    """Test Astakos functionality"""
+    def test_001_unauthorized_access(self):
+        """Test that access without a valid token fails"""
+        false_token = "12345"
+        self.info("Will use token %s", false_token)
+        client = ComputeClient(self.clients.compute_url, false_token)
+        client.CONNECTION_RETRY_LIMIT = self.clients.retry
+
+        with self.assertRaises(ClientError) as cl_error:
+            client.list_servers()
+            self.assertEqual(cl_error.exception.status, 401)
+
+    def test_002_name2uuid(self):
+        """Test that usernames2uuids and uuids2usernames are complementary"""
+        our_uuid = self._get_uuid()
+
+        given_name = self.clients.astakos.uuids2usernames([our_uuid])
+        self.info("uuids2usernames returned %s", given_name)
+        self.assertIn(our_uuid, given_name)
+
+        given_uuid = \
+            self.clients.astakos.usernames2uuids([given_name[our_uuid]])
+        self.info("usernames2uuids returned %s", given_uuid)
+        self.assertIn(given_name[our_uuid], given_uuid)
+
+        self.assertEqual(given_uuid[given_name[our_uuid]], our_uuid)

snf-tools/synnefo_tools/burnin/filelocker.py

+# filelocker.py - Cross-platform (posix/nt) API for flock-style file locking.
+#                 Requires python 1.5.2 or better.
+"""Cross-platform (posix/nt) API for flock-style file locking.
+
+
+Synopsis:
+
+    import filelocker
+    with filelocker.lock("lockfile", filelocker.LOCK_EX):
+        print "Got it"
+
+
+Methods:
+
+   lock ( file, flags, tries=10 )
+
+
+Constants: