burnin.py 80.6 KB
Newer Older
1
#!/usr/bin/env python
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

# Copyright 2011 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

"""Perform integration testing on a running Synnefo deployment"""

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
38
#import __main__
39
40
41
42
import datetime
import inspect
import logging
import os
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
43
import os.path
44
import paramiko
45
import prctl
46
import subprocess
47
import signal
48
49
50
import socket
import sys
import time
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
51
import tempfile
52
from base64 import b64encode
53
from IPy import IP
54
from multiprocessing import Process, Queue
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
55
from random import choice, randint
John Giannelos's avatar
John Giannelos committed
56
from optparse import OptionParser, OptionValueError
57

John Giannelos's avatar
John Giannelos committed
58
59
from kamaki.clients.compute import ComputeClient
from kamaki.clients.cyclades import CycladesClient
60
from kamaki.clients.image import ImageClient
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
61
from kamaki.clients.pithos import PithosClient
62
from kamaki.clients.astakos import AstakosClient
John Giannelos's avatar
John Giannelos committed
63
from kamaki.clients import ClientError
John Giannelos's avatar
John Giannelos committed
64

65
from vncauthproxy.d3des import generate_response as d3des_generate_response
66
67
68
69
70

# Use backported unittest functionality if Python < 2.7
try:
    import unittest2 as unittest
except ImportError:
71
72
    if sys.version_info < (2, 7):
        raise Exception("The unittest2 package is required for Python < 2.7")
73
74
    import unittest

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
75
76
# --------------------------------------------------------------------
# Global Variables
77
AUTH_URL = None
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
TOKEN = None
PLANKTON_USER = None
NO_IPV6 = None
DEFAULT_PLANKTON_USER = "images@okeanos.grnet.gr"
NOFAILFAST = None
VERBOSE = None

# A unique id identifying this test run
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
                                         "%Y%m%d%H%M%S")
SNF_TEST_PREFIX = "snf-test-"

red = '\x1b[31m'
yellow = '\x1b[33m'
green = '\x1b[32m'
normal = '\x1b[0m'

95

96
97
98
99
100
101
102
103
# --------------------------------------------------------------------
# Global functions
def _ssh_execute(hostip, username, password, command):
    """Execute a command via ssh"""
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    try:
        ssh.connect(hostip, username=username, password=password)
104
105
    except socket.error, err:
        raise AssertionError(err)
106
107
    try:
        stdin, stdout, stderr = ssh.exec_command(command)
108
109
    except paramiko.SSHException, err:
        raise AssertionError(err)
110
111
112
113
114
115
    status = stdout.channel.recv_exit_status()
    output = stdout.readlines()
    ssh.close()
    return output, status


116
117
def _get_user_id():
    """Authenticate to astakos and get unique users id"""
118
    astakos = AstakosClient(AUTH_URL, TOKEN)
119
    authenticate = astakos.authenticate()
120
    return authenticate['access']['user']['id']
121
122


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
123
124
# --------------------------------------------------------------------
# BurninTestReulst class
125
126
127
128
class BurninTestResult(unittest.TextTestResult):
    def addSuccess(self, test):
        super(BurninTestResult, self).addSuccess(test)
        if self.showAll:
John Giannelos's avatar
John Giannelos committed
129
            if hasattr(test, 'result_dict'):
130
131
132
133
134
135
136
137
138
                run_details = test.result_dict

                self.stream.write("\n")
                for i in run_details:
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
                self.stream.write("\n")

        elif self.dots:
            self.stream.write('.')
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
139
140
            self.stream.flush()

141
142
143
144
    def addError(self, test, err):
        super(BurninTestResult, self).addError(test, err)
        if self.showAll:
            self.stream.writeln("ERROR")
John Giannelos's avatar
John Giannelos committed
145
146
            if hasattr(test, 'result_dict'):
                run_details = test.result_dict
147

John Giannelos's avatar
John Giannelos committed
148
149
150
151
                self.stream.write("\n")
                for i in run_details:
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
                self.stream.write("\n")
152
153
154
155
156
157
158
159
160

        elif self.dots:
            self.stream.write('E')
            self.stream.flush()

    def addFailure(self, test, err):
        super(BurninTestResult, self).addFailure(test, err)
        if self.showAll:
            self.stream.writeln("FAIL")
John Giannelos's avatar
John Giannelos committed
161
162
            if hasattr(test, 'result_dict'):
                run_details = test.result_dict
163

John Giannelos's avatar
John Giannelos committed
164
165
166
167
                self.stream.write("\n")
                for i in run_details:
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
                self.stream.write("\n")
168
169
170
171
172
173

        elif self.dots:
            self.stream.write('F')
            self.stream.flush()


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
174
175
# --------------------------------------------------------------------
# Format Results
John Giannelos's avatar
John Giannelos committed
176
177
class burninFormatter(logging.Formatter):
    err_fmt = red + "ERROR: %(msg)s" + normal
John Giannelos's avatar
John Giannelos committed
178
    dbg_fmt = green + "* %(msg)s" + normal
John Giannelos's avatar
John Giannelos committed
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    info_fmt = "%(msg)s"

    def __init__(self, fmt="%(levelno)s: %(msg)s"):
        logging.Formatter.__init__(self, fmt)

    def format(self, record):
        format_orig = self._fmt
        # Replace the original format with one customized by logging level
        if record.levelno == 10:    # DEBUG
            self._fmt = burninFormatter.dbg_fmt
        elif record.levelno == 20:  # INFO
            self._fmt = burninFormatter.info_fmt
        elif record.levelno == 40:  # ERROR
            self._fmt = burninFormatter.err_fmt
        result = logging.Formatter.format(self, record)
        self._fmt = format_orig
        return result

197
log = logging.getLogger("burnin")
John Giannelos's avatar
John Giannelos committed
198
199
200
201
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setFormatter(burninFormatter())
log.addHandler(handler)
202

203

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
204
205
# --------------------------------------------------------------------
# UnauthorizedTestCase class
206
class UnauthorizedTestCase(unittest.TestCase):
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
207
    """Test unauthorized access"""
208
209
    @classmethod
    def setUpClass(cls):
210
211
212
        cls.astakos = AstakosClient(AUTH_URL, TOKEN)
        cls.compute_url = \
            cls.astakos.get_service_endpoints('compute')['publicURL']
213
214
        cls.result_dict = dict()

215
216
    def test_unauthorized_access(self):
        """Test access without a valid token fails"""
217
        log.info("Authentication test")
218
        falseToken = '12345'
219
        c = ComputeClient(self.compute_url, falseToken)
220

221
222
        with self.assertRaises(ClientError) as cm:
            c.list_servers()
John Giannelos's avatar
John Giannelos committed
223
            self.assertEqual(cm.exception.status, 401)
224
225


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
226
# --------------------------------------------------------------------
227
# This class gest replicated into Images TestCases dynamically
228
229
230
231
232
233
class ImagesTestCase(unittest.TestCase):
    """Test image lists for consistency"""
    @classmethod
    def setUpClass(cls):
        """Initialize kamaki, get (detailed) list of images"""
        log.info("Getting simple and detailed list of images")
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Compute Client
        compute_url = \
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
        cls.compute_client = ComputeClient(compute_url, TOKEN)
        # Image Client
        image_url = \
            cls.astakos_client.get_service_endpoints('image')['publicURL']
        cls.image_client = ImageClient(image_url, TOKEN)
        # Pithos Client
        pithos_url = cls.astakos_client.\
            get_service_endpoints('object-store')['publicURL']
        cls.pithos_client = PithosClient(pithos_url, TOKEN)

        # Get images
249
250
        cls.images = \
            filter(lambda x: not x['name'].startswith(SNF_TEST_PREFIX),
251
                   cls.image_client.list_public())
252
253
        cls.dimages = \
            filter(lambda x: not x['name'].startswith(SNF_TEST_PREFIX),
254
                   cls.image_client.list_public(detail=True))
255
        cls.result_dict = dict()
256
257
258
        # Get uniq user id
        cls.uuid = _get_user_id()
        log.info("Uniq user id = %s" % cls.uuid)
259
260
261
262
263
264
        # Create temp directory and store it inside our class
        # XXX: In my machine /tmp has not enough space
        #      so use current directory to be sure.
        cls.temp_dir = tempfile.mkdtemp(dir=os.getcwd())
        cls.temp_image_name = \
            SNF_TEST_PREFIX + cls.imageid + ".diskdump"
265

266
267
268
269
270
271
272
273
274
275
276
277
278
    @classmethod
    def tearDownClass(cls):
        """Remove local files"""
        try:
            temp_file = os.path.join(cls.temp_dir, cls.temp_image_name)
            os.unlink(temp_file)
        except:
            pass
        try:
            os.rmdir(cls.temp_dir)
        except:
            pass

279
280
    def test_001_list_images(self):
        """Test image list actually returns images"""
John Giannelos's avatar
John Giannelos committed
281
        self.assertGreater(len(self.images), 0)
282

283
284
    def test_002_list_images_detailed(self):
        """Test detailed image list is the same length as list"""
John Giannelos's avatar
John Giannelos committed
285
        self.assertEqual(len(self.dimages), len(self.images))
286

287
288
289
290
291
292
    def test_003_same_image_names(self):
        """Test detailed and simple image list contain same names"""
        names = sorted(map(lambda x: x["name"], self.images))
        dnames = sorted(map(lambda x: x["name"], self.dimages))
        self.assertEqual(names, dnames)

293
294
295
296
297
298
299
300
# XXX: Find a way to resolve owner's uuid to username.
#      (maybe use astakosclient)
#    def test_004_unique_image_names(self):
#        """Test system images have unique names"""
#        sys_images = filter(lambda x: x['owner'] == PLANKTON_USER,
#                            self.dimages)
#        names = sorted(map(lambda x: x["name"], sys_images))
#        self.assertEqual(sorted(list(set(names))), names)
301
302
303

    def test_005_image_metadata(self):
        """Test every image has specific metadata defined"""
304
        keys = frozenset(["osfamily", "root_partition"])
305
        details = self.compute_client.list_images(detail=True)
John Giannelos's avatar
John Giannelos committed
306
        for i in details:
307
            self.assertTrue(keys.issubset(i["metadata"].keys()))
308

309
310
311
312
313
314
315
316
317
    def test_006_download_image(self):
        """Download image from pithos+"""
        # Get image location
        image = filter(
            lambda x: x['id'] == self.imageid, self.dimages)[0]
        image_location = \
            image['location'].replace("://", " ").replace("/", " ").split()
        log.info("Download image, with owner %s\n\tcontainer %s, and name %s"
                 % (image_location[1], image_location[2], image_location[3]))
318
319
        self.pithos_client.account = image_location[1]
        self.pithos_client.container = image_location[2]
320
321
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
        with open(temp_file, "wb+") as f:
322
            self.pithos_client.download_object(image_location[3], f)
323
324
325
326
327
328

    def test_007_upload_image(self):
        """Upload and register image"""
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
        log.info("Upload image to pithos+")
        # Create container `images'
329
330
331
        self.pithos_client.account = self.uuid
        self.pithos_client.container = "images"
        self.pithos_client.container_put()
332
        with open(temp_file, "rb+") as f:
333
            self.pithos_client.upload_object(self.temp_image_name, f)
334
        log.info("Register image to plankton")
335
        location = "pithos://" + self.uuid + \
336
337
338
            "/images/" + self.temp_image_name
        params = {'is_public': True}
        properties = {'OSFAMILY': "linux", 'ROOT_PARTITION': 1}
339
340
        self.image_client.register(
            self.temp_image_name, location, params, properties)
341
        # Get image id
342
        details = self.image_client.list_public(detail=True)
343
        detail = filter(lambda x: x['location'] == location, details)
344
345
346
347
348
349
350
351
352
        self.assertEqual(len(detail), 1)
        cls = type(self)
        cls.temp_image_id = detail[0]['id']
        log.info("Image registered with id %s" % detail[0]['id'])

    def test_008_cleanup_image(self):
        """Cleanup image test"""
        log.info("Cleanup image test")
        # Remove image from pithos+
353
354
355
        self.pithos_client.account = self.uuid
        self.pithos_client.container = "images"
        self.pithos_client.del_object(self.temp_image_name)
356

357

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
358
359
# --------------------------------------------------------------------
# FlavorsTestCase class
360
361
362
363
364
365
class FlavorsTestCase(unittest.TestCase):
    """Test flavor lists for consistency"""
    @classmethod
    def setUpClass(cls):
        """Initialize kamaki, get (detailed) list of flavors"""
        log.info("Getting simple and detailed list of flavors")
366
367
368
369
370
371
372
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Compute Client
        compute_url = \
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
        cls.compute_client = ComputeClient(compute_url, TOKEN)
        cls.flavors = cls.compute_client.list_flavors()
        cls.dflavors = cls.compute_client.list_flavors(detail=True)
373
        cls.result_dict = dict()
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398

    def test_001_list_flavors(self):
        """Test flavor list actually returns flavors"""
        self.assertGreater(len(self.flavors), 0)

    def test_002_list_flavors_detailed(self):
        """Test detailed flavor list is the same length as list"""
        self.assertEquals(len(self.dflavors), len(self.flavors))

    def test_003_same_flavor_names(self):
        """Test detailed and simple flavor list contain same names"""
        names = sorted(map(lambda x: x["name"], self.flavors))
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
        self.assertEqual(names, dnames)

    def test_004_unique_flavor_names(self):
        """Test flavors have unique names"""
        names = sorted(map(lambda x: x["name"], self.flavors))
        self.assertEqual(sorted(list(set(names))), names)

    def test_005_well_formed_flavor_names(self):
        """Test flavors have names of the form CxxRyyDzz
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
        """
        for f in self.dflavors:
399
            flavor = (f["vcpus"], f["ram"], f["disk"], f["SNF:disk_template"])
Christos Stavrakakis's avatar
Christos Stavrakakis committed
400
            self.assertEqual("C%dR%dD%d%s" % flavor,
401
402
403
404
                             f["name"],
                             "Flavor %s does not match its specs." % f["name"])


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
405
406
# --------------------------------------------------------------------
# ServersTestCase class
407
408
409
410
411
412
class ServersTestCase(unittest.TestCase):
    """Test server lists for consistency"""
    @classmethod
    def setUpClass(cls):
        """Initialize kamaki, get (detailed) list of servers"""
        log.info("Getting simple and detailed list of servers")
413

414
415
416
417
418
419
420
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Compute Client
        compute_url = \
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
        cls.compute_client = ComputeClient(compute_url, TOKEN)
        cls.servers = cls.compute_client.list_servers()
        cls.dservers = cls.compute_client.list_servers(detail=True)
421
        cls.result_dict = dict()
422

423
424
425
    # def test_001_list_servers(self):
    #     """Test server list actually returns servers"""
    #     self.assertGreater(len(self.servers), 0)
426
427
428
429
430
431
432
433
434
435
436
437

    def test_002_list_servers_detailed(self):
        """Test detailed server list is the same length as list"""
        self.assertEqual(len(self.dservers), len(self.servers))

    def test_003_same_server_names(self):
        """Test detailed and simple flavor list contain same names"""
        names = sorted(map(lambda x: x["name"], self.servers))
        dnames = sorted(map(lambda x: x["name"], self.dservers))
        self.assertEqual(names, dnames)


Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
438
439
440
441
442
443
444
# --------------------------------------------------------------------
# Pithos Test Cases
class PithosTestCase(unittest.TestCase):
    """Test pithos functionality"""
    @classmethod
    def setUpClass(cls):
        """Initialize kamaki, get list of containers"""
445
446
447
        # Get uniq user id
        cls.uuid = _get_user_id()
        log.info("Uniq user id = %s" % cls.uuid)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
448
        log.info("Getting list of containers")
449
450
451
452
453
454
455
456

        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Pithos Client
        pithos_url = cls.astakos_client.\
            get_service_endpoints('object-store')['publicURL']
        cls.pithos_client = PithosClient(pithos_url, TOKEN, cls.uuid)

        cls.containers = cls.pithos_client.list_containers()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
        cls.result_dict = dict()

    def test_001_list_containers(self):
        """Test container list actually returns containers"""
        self.assertGreater(len(self.containers), 0)

    def test_002_unique_containers(self):
        """Test if containers have unique names"""
        names = [n['name'] for n in self.containers]
        names = sorted(names)
        self.assertEqual(sorted(list(set(names))), names)

    def test_003_create_container(self):
        """Test create a container"""
        rand_num = randint(1000, 9999)
        rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
        names = [n['name'] for n in self.containers]
        while rand_name in names:
            rand_num = randint(1000, 9999)
            rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
        # Create container
478
479
        self.pithos_client.container = rand_name
        self.pithos_client.container_put()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
480
        # Get list of containers
481
        new_containers = self.pithos_client.list_containers()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
482
483
484
485
486
487
488
489
490
491
        new_container_names = [n['name'] for n in new_containers]
        self.assertIn(rand_name, new_container_names)

    def test_004_upload(self):
        """Test uploading something to pithos+"""
        # Create a tmp file
        with tempfile.TemporaryFile() as f:
            f.write("This is a temp file")
            f.seek(0, 0)
            # Where to save file
492
            self.pithos_client.upload_object("test.txt", f)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
493
494
495
496
497
498
499

    def test_005_download(self):
        """Test download something from pithos+"""
        # Create tmp directory to save file
        tmp_dir = tempfile.mkdtemp()
        tmp_file = os.path.join(tmp_dir, "test.txt")
        with open(tmp_file, "wb+") as f:
500
            self.pithos_client.download_object("test.txt", f)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
501
502
503
504
505
506
507
508
509
510
511
            # Read file
            f.seek(0, 0)
            content = f.read()
        # Remove files
        os.unlink(tmp_file)
        os.rmdir(tmp_dir)
        # Compare results
        self.assertEqual(content, "This is a temp file")

    def test_006_remove(self):
        """Test removing files and containers"""
512
513
514
        cont_name = self.pithos_client.container
        self.pithos_client.del_object("test.txt")
        self.pithos_client.purge_container()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
515
        # List containers
516
        containers = self.pithos_client.list_containers()
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
517
518
519
520
521
        cont_names = [n['name'] for n in containers]
        self.assertNotIn(cont_name, cont_names)


# --------------------------------------------------------------------
522
523
524
525
526
527
# This class gets replicated into actual TestCases dynamically
class SpawnServerTestCase(unittest.TestCase):
    """Test scenario for server of the specified image"""
    @classmethod
    def setUpClass(cls):
        """Initialize a kamaki instance"""
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
528
        log.info("Spawning server for image `%s'" % cls.imagename)
529
530
531
532
533
534
535

        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Cyclades Client
        compute_url = \
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
        cls.cyclades_client = CycladesClient(compute_url, TOKEN)

536
        cls.result_dict = dict()
537
538

    def _get_ipv4(self, server):
539
        """Get the public IPv4 of a server from the detailed server info"""
540

541
        nics = server["attachments"]
John Giannelos's avatar
John Giannelos committed
542

543
544
        for nic in nics:
            net_id = nic["network_id"]
545
            if self.cyclades_client.get_network_details(net_id)["public"]:
546
                public_addrs = nic["ipv4"]
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
547
548

        self.assertTrue(public_addrs is not None)
John Giannelos's avatar
John Giannelos committed
549

550
        return public_addrs
551
552

    def _get_ipv6(self, server):
553
        """Get the public IPv6 of a server from the detailed server info"""
John Giannelos's avatar
John Giannelos committed
554

555
        nics = server["attachments"]
John Giannelos's avatar
John Giannelos committed
556

557
558
        for nic in nics:
            net_id = nic["network_id"]
559
            if self.cyclades_client.get_network_details(net_id)["public"]:
560
                public_addrs = nic["ipv6"]
John Giannelos's avatar
John Giannelos committed
561

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
562
        self.assertTrue(public_addrs is not None)
John Giannelos's avatar
John Giannelos committed
563

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
564
        return public_addrs
565

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
566
    def _connect_loginname(self, os_value):
567
        """Return the login name for connections based on the server OS"""
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
568
        if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
569
            return "user"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
570
        elif os_value in ("windows", "windows_alpha1"):
571
            return "Administrator"
572
        else:
573
            return "root"
574
575
576

    def _verify_server_status(self, current_status, new_status):
        """Verify a server has switched to a specified status"""
577
        server = self.cyclades_client.get_server_details(self.serverid)
578
579
        if server["status"] not in (current_status, new_status):
            return None  # Do not raise exception, return so the test fails
580
581
582
583
584
585
586
587
588
589
590
        self.assertEquals(server["status"], new_status)

    def _get_connected_tcp_socket(self, family, host, port):
        """Get a connected socket from the specified family to host:port"""
        sock = None
        for res in \
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
                               socket.AI_PASSIVE):
            af, socktype, proto, canonname, sa = res
            try:
                sock = socket.socket(af, socktype, proto)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
591
            except socket.error:
592
593
594
595
                sock = None
                continue
            try:
                sock.connect(sa)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
596
            except socket.error:
597
598
599
600
601
602
603
604
605
606
607
608
609
610
                sock.close()
                sock = None
                continue
        self.assertIsNotNone(sock)
        return sock

    def _ping_once(self, ipv6, ip):
        """Test server responds to a single IPv4 or IPv6 ping"""
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
        ping = subprocess.Popen(cmd, shell=True,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (stdout, stderr) = ping.communicate()
        ret = ping.wait()
        self.assertEquals(ret, 0)
611

612
    def _get_hostname_over_ssh(self, hostip, username, password):
613
614
        lines, status = _ssh_execute(
            hostip, username, password, "hostname")
615
        self.assertEqual(len(lines), 1)
616
        return lines[0]
617
618
619
620

    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
                                   opmsg, callable, *args, **kwargs):
        if warn_timeout == fail_timeout:
621
622
623
624
            warn_timeout = fail_timeout + 1
        warn_tmout = time.time() + warn_timeout
        fail_tmout = time.time() + fail_timeout
        while True:
625
            self.assertLess(time.time(), fail_tmout,
626
                            "operation `%s' timed out" % opmsg)
627
            if time.time() > warn_tmout:
628
629
                log.warning("Server %d: `%s' operation `%s' not done yet",
                            self.serverid, self.servername, opmsg)
630
            try:
631
                log.info("%s... " % opmsg)
632
633
634
                return callable(*args, **kwargs)
            except AssertionError:
                pass
635
636
            time.sleep(self.query_interval)

637
    def _insist_on_tcp_connection(self, family, host, port):
638
639
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
640
641
642
        msg = "connect over %s to %s:%s" % \
              (familystr.get(family, "Unknown"), host, port)
        sock = self._try_until_timeout_expires(
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
643
644
645
            self.action_timeout, self.action_timeout,
            msg, self._get_connected_tcp_socket,
            family, host, port)
646
647
648
        return sock

    def _insist_on_status_transition(self, current_status, new_status,
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
649
                                     fail_timeout, warn_timeout=None):
650
651
        msg = "Server %d: `%s', waiting for %s -> %s" % \
              (self.serverid, self.servername, current_status, new_status)
652
653
654
655
656
        if warn_timeout is None:
            warn_timeout = fail_timeout
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
                                        msg, self._verify_server_status,
                                        current_status, new_status)
657
        # Ensure the status is actually the expected one
658
        server = self.cyclades_client.get_server_details(self.serverid)
659
        self.assertEquals(server["status"], new_status)
660
661

    def _insist_on_ssh_hostname(self, hostip, username, password):
662
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
663
        hostname = self._try_until_timeout_expires(
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
664
665
666
            self.action_timeout, self.action_timeout,
            msg, self._get_hostname_over_ssh,
            hostip, username, password)
667
668
669

        # The hostname must be of the form 'prefix-id'
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
670

671
672
673
674
    def _check_file_through_ssh(self, hostip, username, password,
                                remotepath, content):
        msg = "Trying file injection through SSH to %s, as %s/%s" % \
            (hostip, username, password)
675
        log.info(msg)
676
677
678
679
        try:
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(hostip, username=username, password=password)
680
            ssh.close()
681
682
        except socket.error, err:
            raise AssertionError(err)
683

684
685
686
687
        transport = paramiko.Transport((hostip, 22))
        transport.connect(username=username, password=password)

        localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection'
688
        sftp = paramiko.SFTPClient.from_transport(transport)
689
        sftp.get(remotepath, localpath)
690
691
692
        sftp.close()
        transport.close()

693
694
695
        f = open(localpath)
        remote_content = b64encode(f.read())

696
        # Check if files are the same
697
        return (remote_content == content)
698

699
700
701
702
703
704
    def _skipIf(self, condition, msg):
        if condition:
            self.skipTest(msg)

    def test_001_submit_create_server(self):
        """Test submit create server request"""
705
706

        log.info("Submit new server request")
707
708
        server = self.cyclades_client.create_server(
            self.servername, self.flavorid, self.imageid, self.personality)
709

710
711
        log.info("Server id: " + str(server["id"]))
        log.info("Server password: " + server["adminPass"])
712
        self.assertEqual(server["name"], self.servername)
713
714
        self.assertEqual(server["flavor"], self.flavorid)
        self.assertEqual(server["image"], self.imageid)
715
716
717
718
719
        self.assertEqual(server["status"], "BUILD")

        # Update class attributes to reflect data on building server
        cls = type(self)
        cls.serverid = server["id"]
720
        cls.username = None
721
722
        cls.passwd = server["adminPass"]

723
724
725
        self.result_dict["Server ID"] = str(server["id"])
        self.result_dict["Password"] = str(server["adminPass"])

726
727
    def test_002a_server_is_building_in_list(self):
        """Test server is in BUILD state, in server list"""
728
729
        log.info("Server in BUILD state in server list")

730
731
        self.result_dict.clear()

732
        servers = self.cyclades_client.list_servers(detail=True)
733
        servers = filter(lambda x: x["name"] == self.servername, servers)
John Giannelos's avatar
John Giannelos committed
734

735
736
        server = servers[0]
        self.assertEqual(server["name"], self.servername)
737
738
        self.assertEqual(server["flavor"], self.flavorid)
        self.assertEqual(server["image"], self.imageid)
739
740
741
742
        self.assertEqual(server["status"], "BUILD")

    def test_002b_server_is_building_in_details(self):
        """Test server is in BUILD state, in details"""
743
744
745

        log.info("Server in BUILD state in details")

746
        server = self.cyclades_client.get_server_details(self.serverid)
747
        self.assertEqual(server["name"], self.servername)
748
749
        self.assertEqual(server["flavor"], self.flavorid)
        self.assertEqual(server["image"], self.imageid)
750
751
752
        self.assertEqual(server["status"], "BUILD")

    def test_002c_set_server_metadata(self):
753
754
755

        log.info("Creating server metadata")

756
        image = self.cyclades_client.get_image_details(self.imageid)
757
758
        os_value = image["metadata"]["os"]
        users = image["metadata"].get("users", None)
759
        self.cyclades_client.update_server_metadata(self.serverid, OS=os_value)
John Giannelos's avatar
John Giannelos committed
760

761
        userlist = users.split()
762

763
764
765
        # Determine the username to use for future connections
        # to this host
        cls = type(self)
766
767
768

        if "root" in userlist:
            cls.username = "root"
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
769
770
        elif users is None:
            cls.username = self._connect_loginname(os_value)
771
772
773
        else:
            cls.username = choice(userlist)

774
        self.assertIsNotNone(cls.username)
775
776
777

    def test_002d_verify_server_metadata(self):
        """Test server metadata keys are set based on image metadata"""
778
779
780

        log.info("Verifying image metadata")

781
782
        servermeta = self.cyclades_client.get_server_metadata(self.serverid)
        imagemeta = self.cyclades_client.get_image_metadata(self.imageid)
783

John Giannelos's avatar
John Giannelos committed
784
        self.assertEqual(servermeta["OS"], imagemeta["os"])
785
786
787

    def test_003_server_becomes_active(self):
        """Test server becomes ACTIVE"""
788
789
790

        log.info("Waiting for server to become ACTIVE")

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
791
792
        self._insist_on_status_transition(
            "BUILD", "ACTIVE", self.build_fail, self.build_warning)
793

John Giannelos's avatar
John Giannelos committed
794
795
    def test_003a_get_server_oob_console(self):
        """Test getting OOB server console over VNC
796

John Giannelos's avatar
John Giannelos committed
797
798
        Implementation of RFB protocol follows
        http://www.realvnc.com/docs/rfbproto.pdf.
799

John Giannelos's avatar
John Giannelos committed
800
        """
801
        console = self.cyclades_client.get_server_console(self.serverid)
John Giannelos's avatar
John Giannelos committed
802
        self.assertEquals(console['type'], "vnc")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
803
804
        sock = self._insist_on_tcp_connection(
            socket.AF_INET, console["host"], console["port"])
John Giannelos's avatar
John Giannelos committed
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830

        # Step 1. ProtocolVersion message (par. 6.1.1)
        version = sock.recv(1024)
        self.assertEquals(version, 'RFB 003.008\n')
        sock.send(version)

        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
        sec = sock.recv(1024)
        self.assertEquals(list(sec), ['\x01', '\x02'])

        # Step 3. Request VNC Authentication (par 6.1.2)
        sock.send('\x02')

        # Step 4. Receive Challenge (par 6.2.2)
        challenge = sock.recv(1024)
        self.assertEquals(len(challenge), 16)

        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
        response = d3des_generate_response(
            (console["password"] + '\0' * 8)[:8], challenge)
        sock.send(response)

        # Step 6. SecurityResult (par 6.1.3)
        result = sock.recv(4)
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
        sock.close()
831

832
833
    def test_004_server_has_ipv4(self):
        """Test active server has a valid IPv4 address"""
834

835
        log.info("Validate server's IPv4")
836

837
        server = self.cyclades_client.get_server_details(self.serverid)
838
        ipv4 = self._get_ipv4(server)
839
840
841
842

        self.result_dict.clear()
        self.result_dict["IPv4"] = str(ipv4)

843
844
        self.assertEquals(IP(ipv4).version(), 4)

845
846
    def test_005_server_has_ipv6(self):
        """Test active server has a valid IPv6 address"""
847
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
848

849
        log.info("Validate server's IPv6")
850

851
        server = self.cyclades_client.get_server_details(self.serverid)
852
        ipv6 = self._get_ipv6(server)
853
854
855
856

        self.result_dict.clear()
        self.result_dict["IPv6"] = str(ipv6)

857
        self.assertEquals(IP(ipv6).version(), 6)
858

John Giannelos's avatar
John Giannelos committed
859
860
861
862
    def test_006_server_responds_to_ping_IPv4(self):
        """Test server responds to ping on IPv4 address"""

        log.info("Testing if server responds to pings in IPv4")
863
        self.result_dict.clear()
John Giannelos's avatar
John Giannelos committed
864

865
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
866
867
868
869
870
871
872
873
874
875
876
877
        ip = self._get_ipv4(server)
        self._try_until_timeout_expires(self.action_timeout,
                                        self.action_timeout,
                                        "PING IPv4 to %s" % ip,
                                        self._ping_once,
                                        False, ip)

    def test_007_server_responds_to_ping_IPv6(self):
        """Test server responds to ping on IPv6 address"""
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
        log.info("Testing if server responds to pings in IPv6")

878
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
879
880
881
882
883
884
        ip = self._get_ipv6(server)
        self._try_until_timeout_expires(self.action_timeout,
                                        self.action_timeout,
                                        "PING IPv6 to %s" % ip,
                                        self._ping_once,
                                        True, ip)
885
886
887

    def test_008_submit_shutdown_request(self):
        """Test submit request to shutdown server"""
888
889
890

        log.info("Shutting down server")

891
        self.cyclades_client.shutdown_server(self.serverid)
892
893
894

    def test_009_server_becomes_stopped(self):
        """Test server becomes STOPPED"""
895
896

        log.info("Waiting until server becomes STOPPED")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
897
898
        self._insist_on_status_transition(
            "ACTIVE", "STOPPED", self.action_timeout, self.action_timeout)
899
900
901

    def test_010_submit_start_request(self):
        """Test submit start server request"""
902
903
904

        log.info("Starting server")

905
        self.cyclades_client.start_server(self.serverid)
906
907
908

    def test_011_server_becomes_active(self):
        """Test server becomes ACTIVE again"""
909
910

        log.info("Waiting until server becomes ACTIVE")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
911
912
        self._insist_on_status_transition(
            "STOPPED", "ACTIVE", self.action_timeout, self.action_timeout)
913

John Giannelos's avatar
John Giannelos committed
914
915
    def test_011a_server_responds_to_ping_IPv4(self):
        """Test server OS is actually up and running again"""
916

John Giannelos's avatar
John Giannelos committed
917
        log.info("Testing if server is actually up and running")
918

John Giannelos's avatar
John Giannelos committed
919
        self.test_006_server_responds_to_ping_IPv4()
920

John Giannelos's avatar
John Giannelos committed
921
922
    def test_012_ssh_to_server_IPv4(self):
        """Test SSH to server public IPv4 works, verify hostname"""
923

John Giannelos's avatar
John Giannelos committed
924
        self._skipIf(self.is_windows, "only valid for Linux servers")
925
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
926
927
        self._insist_on_ssh_hostname(self._get_ipv4(server),
                                     self.username, self.passwd)
928

John Giannelos's avatar
John Giannelos committed
929
930
931
932
    def test_013_ssh_to_server_IPv6(self):
        """Test SSH to server public IPv6 works, verify hostname"""
        self._skipIf(self.is_windows, "only valid for Linux servers")
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
933

934
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
935
936
        self._insist_on_ssh_hostname(self._get_ipv6(server),
                                     self.username, self.passwd)
937

John Giannelos's avatar
John Giannelos committed
938
939
940
    def test_014_rdp_to_server_IPv4(self):
        "Test RDP connection to server public IPv4 works"""
        self._skipIf(not self.is_windows, "only valid for Windows servers")
941
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
942
        ipv4 = self._get_ipv4(server)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
943
        sock = self._insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
944

John Giannelos's avatar
John Giannelos committed
945
946
947
948
        # No actual RDP processing done. We assume the RDP server is there
        # if the connection to the RDP port is successful.
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
        sock.close()
949

John Giannelos's avatar
John Giannelos committed
950
951
952
953
    def test_015_rdp_to_server_IPv6(self):
        "Test RDP connection to server public IPv6 works"""
        self._skipIf(not self.is_windows, "only valid for Windows servers")
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
954

955
        server = self.cyclades_client.get_server_details(self.serverid)
John Giannelos's avatar
John Giannelos committed
956
        ipv6 = self._get_ipv6(server)
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
957
        sock = self._get_tcp_connection(socket.AF_INET6, ipv6, 3389)
958

John Giannelos's avatar
John Giannelos committed
959
960
961
        # No actual RDP processing done. We assume the RDP server is there
        # if the connection to the RDP port is successful.
        sock.close()
962

John Giannelos's avatar
John Giannelos committed
963
964
965
    def test_016_personality_is_enforced(self):
        """Test file injection for personality enforcement"""
        self._skipIf(self.is_windows, "only implemented for Linux servers")
Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
966
        self._skipIf(self.personality is None, "No personality file selected")
967

John Giannelos's avatar
John Giannelos committed
968
        log.info("Trying to inject file for personality enforcement")
969

970
        server = self.cyclades_client.get_server_details(self.serverid)
971

John Giannelos's avatar
John Giannelos committed
972
973
974
975
976
977
978
        for inj_file in self.personality:
            equal_files = self._check_file_through_ssh(self._get_ipv4(server),
                                                       inj_file['owner'],
                                                       self.passwd,
                                                       inj_file['path'],
                                                       inj_file['contents'])
            self.assertTrue(equal_files)
979

980
981
    def test_017_submit_delete_request(self):
        """Test submit request to delete server"""
982
983
984

        log.info("Deleting server")

985
        self.cyclades_client.delete_server(self.serverid)
986
987
988

    def test_018_server_becomes_deleted(self):
        """Test server becomes DELETED"""
989
990
991

        log.info("Testing if server becomes DELETED")

Ilias Tsitsimpis's avatar
Ilias Tsitsimpis committed
992
993
        self._insist_on_status_transition(
            "ACTIVE", "DELETED", self.action_timeout, self.action_timeout)
994
995
996

    def test_019_server_no_longer_in_server_list(self):
        """Test server is no longer in server list"""
997
998
999

        log.info("Test if server is no longer listed")

1000
        servers = self.cyclades_client.list_servers()
1001
1002
1003
        self.assertNotIn(self.serverid, [s["id"] for s in servers])


1004
class NetworkTestCase(unittest.TestCase):
John Giannelos's avatar
John Giannelos committed
1005
    """ Testing networking in cyclades """
1006

1007
    @classmethod
John Giannelos's avatar
John Giannelos committed
1008
1009
    def setUpClass(cls):
        "Initialize kamaki, get list of current networks"
John Giannelos's avatar
John Giannelos committed
1010

1011
1012
1013
1014
1015
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
        # Cyclades Client
        compute_url = \
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
        cls.cyclades_client = CycladesClient(compute_url, TOKEN)
1016

1017
1018
1019
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
                                          TEST_RUN_ID,
                                          cls.imagename)
1020
1021
1022
1023
1024

        #Dictionary initialization for the vms credentials
        cls.serverid = dict()
        cls.username = dict()
        cls.password = dict()
John Giannelos's avatar
John Giannelos committed
1025
        cls.is_windows = cls.imagename.lower().find("windows") >= 0
1026

1027
1028
        cls.result_dict = dict()

1029
1030
1031
    def _skipIf(self, condition, msg):
        if condition:
            self.skipTest(msg)
1032

1033
1034
1035
    def _get_ipv4(self, server):
        """Get the public IPv4 of a server from the detailed server info"""