pithos-test 87.3 KB
Newer Older
1
#!/usr/bin/env python
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
2 3
#coding=utf8

Antony Chazapis's avatar
Antony Chazapis committed
4
# Copyright 2011-2012 GRNET S.A. All rights reserved.
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
# 
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
# 
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
# 
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# 
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

37 38 39
from pithos.lib.client import Pithos_Client, Fault
from pithos.lib.util import get_user, get_auth, get_server, get_api

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
40
from xml.dom import minidom
41
from StringIO import StringIO
42 43
from hashlib import new as newhasher
from binascii import hexlify
44

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
45
import json
46
import unittest
47
import time as _time
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
48 49 50 51 52
import types
import hashlib
import mimetypes
import random
import datetime
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
53
import string
54 55
import re

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
56 57 58 59
DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
                "%A, %d-%b-%y %H:%M:%S GMT",
                "%a, %d %b %Y %H:%M:%S GMT"]

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
60 61 62 63 64 65 66 67
OTHER_ACCOUNTS = {
    '0001': 'verigak',
    '0002': 'chazapis',
    '0003': 'gtsouk',
    '0004': 'papagian',
    '0005': 'louridas',
    '0006': 'chstath',
    '0007': 'pkanavos',
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
68
    '0008': 'mvasilak',
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
69
    '0009': 'διογένης'}
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
70

71
class BaseTestCase(unittest.TestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
72 73
    #TODO unauthorized request
    def setUp(self):
74 75
        self.client = Pithos_Client(get_server(), get_auth(), get_user(),
                                    get_api())
76
        self._clean_account()
77 78
        self.invalid_client = Pithos_Client(get_server(), get_auth(), 'invalid',
                                            get_api())
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
79 80 81 82 83 84 85
        
        #keep track of initial account groups
        self.initial_groups = self.client.retrieve_account_groups()
        
        #keep track of initial account meta
        self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
86 87 88 89 90
        self.extended = {
            'container':(
                'name',
                'count',
                'bytes',
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
91
                'last_modified',
92
                'x_container_policy'),
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
93 94 95 96 97 98 99
            'object':(
                'name',
                'hash',
                'bytes',
                'content_type',
                'content_encoding',
                'last_modified',)}
100
        self.return_codes = (400, 401, 403, 404, 503,)
101 102
    
    def tearDown(self):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
103 104 105 106 107 108 109 110 111 112 113 114 115
        #delete additionally created meta
        l = []
        for m in self.client.retrieve_account_metadata(restricted=True):
            if m not in self.initial_meta:
                l.append(m)
        self.client.delete_account_metadata(l)
        
        #delete additionally created groups
        l = []
        for g in self.client.retrieve_account_groups():
            if g not in self.initial_groups:
                l.append(g)
        self.client.unset_account_groups(l)
116 117 118
        self._clean_account()
    
    def _clean_account(self):
119
        for c in self.client.list_containers():
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
120 121 122 123 124 125 126 127
            while True:
                #list objects returns at most 10000 objects
                #so repeat until there are no more objects
                objects = self.client.list_objects(c)
                if not objects:
                    break
                for o in objects:
                    self.client.delete_object(c, o)
128 129
            self.client.delete_container(c)
    
130
    def assert_status(self, status, codes):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
131 132 133 134 135
        l = [elem for elem in self.return_codes]
        if type(codes) == types.ListType:
            l.extend(codes)
        else:
            l.append(codes)
136 137
        self.assertTrue(status in l)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
138
    def assert_extended(self, data, format, type, size=10000):
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
        if format == 'xml':
            self._assert_xml(data, type, size)
        elif format == 'json':
            self._assert_json(data, type, size)
    
    def _assert_json(self, data, type, size):
        convert = lambda s: s.lower()
        info = [convert(elem) for elem in self.extended[type]]
        self.assertTrue(len(data) <= size)
        for item in info:
            for i in data:
                if 'subdir' in i.keys():
                    continue
                self.assertTrue(item in i.keys())
    
    def _assert_xml(self, data, type, size):
        convert = lambda s: s.lower()
        info = [convert(elem) for elem in self.extended[type]]
        try:
            info.remove('content_encoding')
        except ValueError:
            pass
        xml = data
        entities = xml.getElementsByTagName(type)
        self.assertTrue(len(entities) <= size)
        for e in entities:
            for item in info:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
166
                self.assertTrue(e.getElementsByTagName(item))
167 168 169 170 171 172 173
    
    def assert_raises_fault(self, status, callableObj, *args, **kwargs):
        """
        asserts that a Fault with a specific status is raised
        when callableObj is called with the specific arguments
        """
        try:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
174
            r = callableObj(*args, **kwargs)
175 176
            self.fail('Should never reach here')
        except Fault, f:
177 178 179 180
            if type(status) == types.ListType:
                self.failUnless(f.status in status)
            else:
                self.failUnless(f.status == status)
181
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
182 183 184 185 186 187 188 189
    def assert_not_raises_fault(self, status, callableObj, *args, **kwargs):
        """
        asserts that a Fault with a specific status is not raised
        when callableObj is called with the specific arguments
        """
        try:
            r = callableObj(*args, **kwargs)
        except Fault, f:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
190
            self.failIfEqual(f.status, status)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
191
    
192 193 194 195 196 197 198 199 200
    def assert_container_exists(self, container):
        """
        asserts the existence of a container
        """
        try:
            self.client.retrieve_container_metadata(container)
        except Fault, f:
            self.failIf(f.status == 404)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
201 202 203 204 205 206 207
    def assert_container_not_exists(self, container):
        """
        asserts there is no such a container
        """
        self.assert_raises_fault(404, self.client.retrieve_container_metadata,
                                 container)
    
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
    def assert_object_exists(self, container, object):
        """
        asserts the existence of an object
        """
        try:
            self.client.retrieve_object_metadata(container, object)
        except Fault, f:
            self.failIf(f.status == 404)
    
    def assert_object_not_exists(self, container, object):
        """
        asserts there is no such an object
        """
        self.assert_raises_fault(404, self.client.retrieve_object_metadata,
                                 container, object)
223
    
224 225 226 227 228 229
    def assert_versionlist_structure(self, versionlist):
        self.assertTrue(type(versionlist) == types.ListType)
        for elem in versionlist:
            self.assertTrue(type(elem) == types.ListType)
            self.assertEqual(len(elem), 2)
    
230 231
    def upload_random_data(self, container, name, length=1024, type=None,
                           enc=None, **meta):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
232
        data = get_random_data(length)
233
        return self.upload_data(container, name, data, type, enc, **meta)
234
    
235 236
    def upload_data(self, container, name, data, type=None, enc=None, etag=None,
                    **meta):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
237 238 239 240
        obj = {}
        obj['name'] = name
        try:
            obj['data'] = data
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
241
            obj['hash'] = compute_md5_hash(obj['data'])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
242
            
243 244 245
            args = {}
            args['etag'] = etag if etag else obj['hash']
            
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
246 247 248 249 250 251
            try:
                guess = mimetypes.guess_type(name)
                type = type if type else guess[0]
                enc = enc if enc else guess[1]
            except:
                pass
252 253 254 255
            args['content_type'] = type if type else 'plain/text'
            args['content_encoding'] = enc if enc else None
            
            obj['meta'] = args
256 257
            
            path = '/%s/%s' % (container, name)
258 259
            self.client.create_object(container, name, f=StringIO(obj['data']),
                                      meta=meta, **args)
260 261
            
            return obj
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
262 263 264
        except IOError:
            return

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
265 266 267 268 269
class AccountHead(BaseTestCase):
    def setUp(self):
        BaseTestCase.setUp(self)
        self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
        for item in self.containers:
270
            self.client.create_container(item)
271 272 273
        
        meta = {'foo':'bar'}
        self.client.update_account_metadata(**meta)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
274
        #self.updated_meta = self.initial_meta.update(meta)
275
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
276
    def test_get_account_meta(self):
277
        meta = self.client.retrieve_account_metadata()
278
        
279
        containers = self.client.list_containers()
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
280
        l = str(len(containers))
281
        self.assertEqual(meta['x-account-container-count'], l)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
282 283
        size = 0
        for c in containers:
284 285 286 287
            m = self.client.retrieve_container_metadata(c)
            size = size + int(m['x-container-bytes-used'])
        self.assertEqual(meta['x-account-bytes-used'], str(size))
    
288 289
    def test_get_account_403(self):
        self.assert_raises_fault(403,
290 291 292 293 294 295 296
                                 self.invalid_client.retrieve_account_metadata)
    
    def test_get_account_meta_until(self):
        t = datetime.datetime.utcnow()
        past = t - datetime.timedelta(minutes=-15)
        past = int(_time.mktime(past.timetuple()))
        
297
        meta = {'premium':True}
298 299 300
        self.client.update_account_metadata(**meta)
        meta = self.client.retrieve_account_metadata(restricted=True,
                                                     until=past)
301
        self.assertTrue('premium' not in meta)
302 303
        
        meta = self.client.retrieve_account_metadata(restricted=True)
304
        self.assertTrue('premium' in meta)
305 306
    
    def test_get_account_meta_until_invalid_date(self):
307
        meta = {'premium':True}
308 309 310
        self.client.update_account_metadata(**meta)
        meta = self.client.retrieve_account_metadata(restricted=True,
                                                     until='kshfksfh')
311
        self.assertTrue('premium' in meta)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
312
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
313
class AccountGet(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
314 315 316 317 318
    def setUp(self):
        BaseTestCase.setUp(self)
        #create some containers
        self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
        for item in self.containers:
319 320
            self.client.create_container(item)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
321 322
    def test_list(self):
        #list containers
323
        containers = self.client.list_containers()
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
324
        self.assertEquals(self.containers, containers)
325
    
326 327
    def test_list_403(self):
        self.assert_raises_fault(403, self.invalid_client.list_containers)
328
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
329 330
    def test_list_with_limit(self):
        limit = 2
331
        containers = self.client.list_containers(limit=limit)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
332 333
        self.assertEquals(len(containers), limit)
        self.assertEquals(self.containers[:2], containers)
334
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
335
    def test_list_with_marker(self):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
336 337
        l = 2
        m = 'bananas'
338
        containers = self.client.list_containers(limit=l, marker=m)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
339 340
        i = self.containers.index(m) + 1
        self.assertEquals(self.containers[i:(i+l)], containers)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
341
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
342
        m = 'oranges'
343
        containers = self.client.list_containers(limit=l, marker=m)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
344 345
        i = self.containers.index(m) + 1
        self.assertEquals(self.containers[i:(i+l)], containers)
346
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
347
    def test_list_json_with_marker(self):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
348 349
        l = 2
        m = 'bananas'
350 351
        containers = self.client.list_containers(limit=l, marker=m, format='json')
        self.assert_extended(containers, 'json', 'container', l)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
352 353
        self.assertEqual(containers[0]['name'], 'kiwis')
        self.assertEqual(containers[1]['name'], 'oranges')
354
    
355 356 357 358
    def test_list_xml_with_marker(self):
        l = 2
        m = 'oranges'
        xml = self.client.list_containers(limit=l, marker=m, format='xml')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
359
        self.assert_extended(xml, 'xml', 'container', l)
360 361 362
        nodes = xml.getElementsByTagName('name')
        self.assertEqual(len(nodes), 1)
        self.assertEqual(nodes[0].childNodes[0].data, 'pears')
363
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
364 365 366 367 368
    def test_if_modified_since(self):
        t = datetime.datetime.utcnow()
        t2 = t - datetime.timedelta(minutes=10)
        
        #add a new container
369 370
        self.client.create_container('dummy')
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
371 372
        for f in DATE_FORMATS:
            past = t2.strftime(f)
373 374 375 376 377 378
            try:
                c = self.client.list_containers(if_modified_since=past)
                self.assertEqual(len(c), len(self.containers) + 1)
            except Fault, f:
                self.failIf(f.status == 304) #fail if not modified
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
379
    def test_if_modified_since_invalid_date(self):
380 381 382
        c = self.client.list_containers(if_modified_since='')
        self.assertEqual(len(c), len(self.containers))
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
383 384 385 386 387
    def test_if_not_modified_since(self):
        now = datetime.datetime.utcnow()
        since = now + datetime.timedelta(1)
        
        for f in DATE_FORMATS:
388 389
            args = {'if_modified_since':'%s' %since.strftime(f)}
            
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
390
            #assert not modified
391 392
            self.assert_raises_fault(304, self.client.list_containers, **args)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
393 394 395 396 397
    def test_if_unmodified_since(self):
        now = datetime.datetime.utcnow()
        since = now + datetime.timedelta(1)
        
        for f in DATE_FORMATS:
398
            c = self.client.list_containers(if_unmodified_since=since.strftime(f))
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
399 400
            
            #assert success
401 402
            self.assertEqual(self.containers, c)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
403 404 405 406 407
    def test_if_unmodified_since_precondition_failed(self):
        t = datetime.datetime.utcnow()
        t2 = t - datetime.timedelta(minutes=10)
        
        #add a new container
408
        self.client.create_container('dummy')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
409 410 411 412
        
        for f in DATE_FORMATS:
            past = t2.strftime(f)
            
413 414
            args = {'if_unmodified_since':'%s' %past}
            
415
            #assert precondition failed
416 417
            self.assert_raises_fault(412, self.client.list_containers, **args)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
418
class AccountPost(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
419 420 421 422
    def setUp(self):
        BaseTestCase.setUp(self)
        self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
        for item in self.containers:
423
            self.client.create_container(item)
424 425
        
        meta = {'foo':'bar'}
426
        self.client.update_account_metadata(**meta)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
427
        self.updated_meta = self.initial_meta.update(meta)
428 429 430 431 432 433 434 435 436 437 438
    
    def test_update_meta(self):
        with AssertMappingInvariant(self.client.retrieve_account_groups):
            meta = {'test':'test', 'tost':'tost'}
            self.client.update_account_metadata(**meta)
            
            meta.update(self.initial_meta)
            self.assertEqual(meta,
                             self.client.retrieve_account_metadata(
                                restricted=True))
        
439
    def test_invalid_account_update_meta(self):
440
        meta = {'test':'test', 'tost':'tost'}
441
        self.assert_raises_fault(403,
442 443
                                 self.invalid_client.update_account_metadata,
                                 **meta)
444 445 446 447 448 449 450 451 452 453 454
    
    def test_reset_meta(self):
        with AssertMappingInvariant(self.client.retrieve_account_groups):
            meta = {'test':'test', 'tost':'tost'}
            self.client.update_account_metadata(**meta)
            
            meta = {'test':'test33'}
            self.client.reset_account_metadata(**meta)
            
            self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True))
    
455 456 457 458 459 460 461 462 463 464
    def test_delete_meta(self):
        with AssertMappingInvariant(self.client.retrieve_account_groups):
            meta = {'test':'test', 'tost':'tost'}
            self.client.update_account_metadata(**meta)
            
            self.client.delete_account_metadata(meta.keys())
            
            account_meta = self.client.retrieve_account_metadata(restricted=True)
            for m in meta:
                self.assertTrue(m not in account_meta.keys())
465 466 467 468 469 470
    
    def test_set_account_groups(self):
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
            groups = {'pithosdev':'verigak,gtsouk,chazapis'}
            self.client.set_account_groups(**groups)
            
Antony Chazapis's avatar
Antony Chazapis committed
471 472
            self.assertEqual(set(groups['pithosdev']),
                             set(self.client.retrieve_account_groups()['pithosdev']))
473 474 475 476 477
            
            more_groups = {'clientsdev':'pkanavos,mvasilak'}
            self.client.set_account_groups(**more_groups)
            
            groups.update(more_groups)
Antony Chazapis's avatar
Antony Chazapis committed
478 479
            self.assertEqual(set(groups['clientsdev']),
                             set(self.client.retrieve_account_groups()['clientsdev']))
480 481 482 483 484 485 486
    
    def test_reset_account_groups(self):
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
            groups = {'pithosdev':'verigak,gtsouk,chazapis',
                      'clientsdev':'pkanavos,mvasilak'}
            self.client.set_account_groups(**groups)
            
Antony Chazapis's avatar
Antony Chazapis committed
487 488 489 490
            self.assertEqual(set(groups['pithosdev'].split(',')),
                             set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
            self.assertEqual(set(groups['clientsdev'].split(',')),
                             set(self.client.retrieve_account_groups()['clientsdev'].split(',')))
491
            
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
492
            groups = {'pithosdev':'verigak,gtsouk,chazapis,papagian'}
493 494
            self.client.reset_account_groups(**groups)
            
Antony Chazapis's avatar
Antony Chazapis committed
495 496
            self.assertEqual(set(groups['pithosdev'].split(',')),
                             set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
497 498 499 500 501 502 503 504 505 506 507
    
    def test_delete_account_groups(self):
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
            groups = {'pithosdev':'verigak,gtsouk,chazapis',
                      'clientsdev':'pkanavos,mvasilak'}
            self.client.set_account_groups(**groups)
            
            self.client.unset_account_groups(groups.keys())
            
            self.assertEqual({}, self.client.retrieve_account_groups())
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
508 509 510 511
class ContainerHead(BaseTestCase):
    def setUp(self):
        BaseTestCase.setUp(self)
        self.container = 'apples'
512 513
        self.client.create_container(self.container)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
514
    def test_get_meta(self):
515
        meta = {'trash':'true'}
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
516
        t1 = datetime.datetime.utcnow()
517
        o = self.upload_random_data(self.container, o_names[0], **meta)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
518
        if o:
519
            headers = self.client.retrieve_container_metadata(self.container)
520 521 522
            self.assertEqual(headers['x-container-object-count'], '1')
            self.assertEqual(headers['x-container-bytes-used'], str(len(o['data'])))
            t2 = datetime.datetime.strptime(headers['last-modified'], DATE_FORMATS[2])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
523 524 525
            delta = (t2 - t1)
            threashold = datetime.timedelta(seconds=1) 
            self.assertTrue(delta < threashold)
526 527
            self.assertTrue(headers['x-container-object-meta'])
            self.assertTrue('Trash' in headers['x-container-object-meta'])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
528 529

class ContainerGet(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
530 531 532 533
    def setUp(self):
        BaseTestCase.setUp(self)
        self.container = ['pears', 'apples']
        for c in self.container:
534
            self.client.create_container(c)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
535 536
        self.obj = []
        for o in o_names[:8]:
537
            self.obj.append(self.upload_random_data(self.container[0], o))
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
538
        for o in o_names[8:]:
539
            self.obj.append(self.upload_random_data(self.container[1], o))
540
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
541
    def test_list_objects(self):
542
        objects = self.client.list_objects(self.container[0])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
543 544 545
        l = [elem['name'] for elem in self.obj[:8]]
        l.sort()
        self.assertEqual(objects, l)
546
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
    def test_list_objects_containing_slash(self):
        self.client.create_container('test')
        self.upload_random_data('test', '/objectname')
        
        objects = self.client.list_objects('test')
        self.assertEqual(objects, ['/objectname'])
        
        objects = self.client.list_objects('test', format='json')
        self.assertEqual(objects[0]['name'], '/objectname')
        
        objects = self.client.list_objects('test', format='xml')
        self.assert_extended(objects, 'xml', 'object')
        node_name = objects.getElementsByTagName('name')[0]
        self.assertEqual(node_name.firstChild.data, '/objectname')

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
562
    def test_list_objects_with_limit_marker(self):
563
        objects = self.client.list_objects(self.container[0], limit=2)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
564 565 566 567 568 569 570 571
        l = [elem['name'] for elem in self.obj[:8]]
        l.sort()
        self.assertEqual(objects, l[:2])
        
        markers = ['How To Win Friends And Influence People.pdf',
                   'moms_birthday.jpg']
        limit = 4
        for m in markers:
572 573
            objects = self.client.list_objects(self.container[0], limit=limit,
                                               marker=m)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
574 575 576 577
            l = [elem['name'] for elem in self.obj[:8]]
            l.sort()
            start = l.index(m) + 1
            end = start + limit
Antony Chazapis's avatar
Antony Chazapis committed
578
            end = end if len(l) >= end else len(l)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
579
            self.assertEqual(objects, l[start:end])
580
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
581
    #takes too long
Antony Chazapis's avatar
Antony Chazapis committed
582
    def _test_list_limit_exceeds(self):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
583 584 585 586 587 588
        self.client.create_container('pithos')
        
        for i in range(10001):
            self.client.create_zero_length_object('pithos', i)
        
        self.assertEqual(10000, len(self.client.list_objects('pithos')))
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
589 590
    
    def test_list_empty_params(self):
591
        objects = self.client.get('/%s/%s' % (get_user(), self.container[0]))[2]
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
592 593 594 595 596
        if objects:
            objects = objects.strip().split('\n')
        self.assertEqual(objects,
                         self.client.list_objects(self.container[0]))
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
597
    def test_list_pseudo_hierarchical_folders(self):
598 599
        objects = self.client.list_objects(self.container[1], prefix='photos',
                                           delimiter='/')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
600 601
        self.assertEquals(['photos/animals/', 'photos/me.jpg',
                           'photos/plants/'], objects)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
602
        
603 604 605
        objects = self.client.list_objects(self.container[1],
                                           prefix='photos/animals',
                                           delimiter='/')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
606
        l = ['photos/animals/cats/', 'photos/animals/dogs/']
607
        self.assertEquals(l, objects)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
608
        
609
        objects = self.client.list_objects(self.container[1], path='photos')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
610
        self.assertEquals(['photos/me.jpg'], objects)
611
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
612
    def test_extended_list_json(self):
613
        objects = self.client.list_objects(self.container[1], format='json',
614 615
                                           limit=2, prefix='photos/animals',
                                           delimiter='/')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
616 617
        self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/')
        self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/')
618
    
619 620 621
    def test_extended_list_xml(self):
        xml = self.client.list_objects(self.container[1], format='xml', limit=4,
                                       prefix='photos', delimiter='/')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
622
        self.assert_extended(xml, 'xml', 'object', size=4)
623 624 625 626 627 628 629 630
        dirs = xml.getElementsByTagName('subdir')
        self.assertEqual(len(dirs), 2)
        self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/')
        self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/')
        
        objects = xml.getElementsByTagName('name')
        self.assertEqual(len(objects), 1)
        self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg')
631
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
632
    def test_list_meta_double_matching(self):
633 634 635
        meta = {'quality':'aaa', 'stock':'true'}
        self.client.update_object_metadata(self.container[0],
                                           self.obj[0]['name'], **meta)
636
        obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
637 638
        self.assertEqual(len(obj), 1)
        self.assertTrue(obj, self.obj[0]['name'])
639
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
640
    def test_list_using_meta(self):
641
        meta = {'quality':'aaa'}
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
642
        for o in self.obj[:2]:
643 644 645
            self.client.update_object_metadata(self.container[0], o['name'],
                                               **meta)
        meta = {'stock':'true'}
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
646
        for o in self.obj[3:5]:
647 648 649
            self.client.update_object_metadata(self.container[0], o['name'],
                                               **meta)
        
650 651 652
        obj = self.client.list_objects(self.container[0], meta='Quality')
        self.assertEqual(len(obj), 2)
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
653 654
        
        # test case insensitive
655
        obj = self.client.list_objects(self.container[0], meta='quality')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
656
        self.assertEqual(len(obj), 2)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
657
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
658 659
        
        # test multiple matches
660
        obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
661
        self.assertEqual(len(obj), 4)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
662
        self.assertTrue(obj, [o['name'] for o in self.obj[:4]])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
663 664
        
        # test non 1-1 multiple match
665
        obj = self.client.list_objects(self.container[0], meta='Quality,aaaa')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
666
        self.assertEqual(len(obj), 2)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
667
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
668
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
669 670 671 672
    def test_if_modified_since(self):
        t = datetime.datetime.utcnow()
        t2 = t - datetime.timedelta(minutes=10)
        
673 674
        #add a new object
        self.upload_random_data(self.container[0], o_names[0])
675
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
676 677
        for f in DATE_FORMATS:
            past = t2.strftime(f)
678 679 680 681 682 683 684 685
            try:
                o = self.client.list_objects(self.container[0],
                                            if_modified_since=past)
                self.assertEqual(o,
                                 self.client.list_objects(self.container[0]))
            except Fault, f:
                self.failIf(f.status == 304) #fail if not modified
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
686
    def test_if_modified_since_invalid_date(self):
687
        headers = {'if-modified-since':''}
688 689 690
        o = self.client.list_objects(self.container[0], if_modified_since='')
        self.assertEqual(o, self.client.list_objects(self.container[0]))
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
691 692 693 694 695
    def test_if_not_modified_since(self):
        now = datetime.datetime.utcnow()
        since = now + datetime.timedelta(1)
        
        for f in DATE_FORMATS:
696 697
            args = {'if_modified_since':'%s' %since.strftime(f)}
            
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
698
            #assert not modified
699 700
            self.assert_raises_fault(304, self.client.list_objects,
                                     self.container[0], **args)
701
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
702 703 704 705 706
    def test_if_unmodified_since(self):
        now = datetime.datetime.utcnow()
        since = now + datetime.timedelta(1)
        
        for f in DATE_FORMATS:
707 708
            obj = self.client.list_objects(self.container[0],
                                           if_unmodified_since=since.strftime(f))
709
            
710 711 712
            #assert unmodified
            self.assertEqual(obj, self.client.list_objects(self.container[0]))
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
713 714 715 716 717
    def test_if_unmodified_since_precondition_failed(self):
        t = datetime.datetime.utcnow()
        t2 = t - datetime.timedelta(minutes=10)
        
        #add a new container
718 719
        self.client.create_container('dummy')
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
720 721 722
        for f in DATE_FORMATS:
            past = t2.strftime(f)
            
723 724
            args = {'if_unmodified_since':'%s' %past}
            
725
            #assert precondition failed
726 727
            self.assert_raises_fault(412, self.client.list_objects,
                                     self.container[0], **args)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
728 729

class ContainerPut(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
730 731 732
    def setUp(self):
        BaseTestCase.setUp(self)
        self.containers = ['c1', 'c2']
733
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
734
    def test_create(self):
735 736
        self.client.create_container(self.containers[0])
        containers = self.client.list_containers()
737
        self.assertTrue(self.containers[0] in containers)
738 739
        self.assert_container_exists(self.containers[0])
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
740
    def test_create_twice(self):
741 742 743
        self.client.create_container(self.containers[0])
        self.assertTrue(not self.client.create_container(self.containers[0]))
    
744 745 746 747 748 749 750 751 752 753 754 755 756
    def test_quota(self):
        self.client.create_container(self.containers[0])
        
        policy = {'quota':100}
        self.client.set_container_policies('c1', **policy)
        
        meta = self.client.retrieve_container_metadata('c1')
        self.assertTrue('x-container-policy-quota' in meta)
        self.assertEqual(meta['x-container-policy-quota'], '100')
        
        args = ['c1', 'o1']
        kwargs = {'length':101}
        self.assert_raises_fault(413, self.upload_random_data, *args, **kwargs)
757 758 759 760
        
        #reset quota
        policy = {'quota':0}
        self.client.set_container_policies('c1', **policy)
761
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
762 763 764 765
class ContainerPost(BaseTestCase):
    def setUp(self):
        BaseTestCase.setUp(self)
        self.container = 'apples'
766 767
        self.client.create_container(self.container)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
768
    def test_update_meta(self):
769 770 771
        meta = {'test':'test33',
                'tost':'tost22'}
        self.client.update_container_metadata(self.container, **meta)
772
        headers = self.client.retrieve_container_metadata(self.container)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
773
        for k,v in meta.items():
774 775 776
            k = 'x-container-meta-%s' % k
            self.assertTrue(headers[k])
            self.assertEqual(headers[k], v)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
777 778

class ContainerDelete(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
779 780 781 782
    def setUp(self):
        BaseTestCase.setUp(self)
        self.containers = ['c1', 'c2']
        for c in self.containers:
783 784
            self.client.create_container(c)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
785
    def test_delete(self):
786 787
        status = self.client.delete_container(self.containers[0])[0]
        self.assertEqual(status, 204)
788
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
789
    def test_delete_non_empty(self):
790
        self.upload_random_data(self.containers[1], o_names[0])
791 792
        self.assert_raises_fault(409, self.client.delete_container,
                                 self.containers[1])
793
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
794
    def test_delete_invalid(self):
795
        self.assert_raises_fault(404, self.client.delete_container, 'c3')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
796

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
797
class ObjectGet(BaseTestCase):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
798 799 800 801 802
    def setUp(self):
        BaseTestCase.setUp(self)
        self.containers = ['c1', 'c2']
        #create some containers
        for c in self.containers:
803
            self.client.create_container(c)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
804 805
        
        #upload a file
806
        names = ('obj1', 'obj2')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
807
        self.objects = []
808 809
        for n in names:
            self.objects.append(self.upload_random_data(self.containers[1], n))
810
    
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
    def test_versions(self):
        c = self.containers[1]
        o = self.objects[0]
        b = self.client.retrieve_object_versionlist(c, o['name'])['versions']
        self.assert_versionlist_structure(b)
        
        #update meta
        meta = {'quality':'AAA', 'stock':True}
        self.client.update_object_metadata(c, o['name'], **meta)
        
        a = self.client.retrieve_object_versionlist(c, o['name'])['versions']
        self.assert_versionlist_structure(a)
        self.assertEqual(len(b)+1, len(a))
        self.assertEqual(b, a[:-1])
        
        #get exact previous version metadata
        v = a[-2][0]
        v_meta = self.client.retrieve_object_metadata(c, o['name'],
                                                      restricted=True,
                                                      version=v)
        for k in meta.keys():
            self.assertTrue(k not in v_meta)
        
        #update obejct
        data = get_random_data()
        self.client.update_object(c, o['name'], StringIO(data))
        
        aa = self.client.retrieve_object_versionlist(c, o['name'])['versions']
        self.assert_versionlist_structure(aa)
        self.assertEqual(len(a)+1, len(aa))
        self.assertEqual(a, aa[:-1])
        
        #get exact previous version
        v = aa[-3][0]
        v_data = self.client.retrieve_object_version(c, o['name'], version=v)
        self.assertEqual(o['data'], v_data)
        self.assertEqual(self.client.retrieve_object(c, o['name']),
                         '%s%s' %(v_data, data))
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
850 851
    def test_get(self):
        #perform get
852 853 854 855 856
        o = self.client.retrieve_object(self.containers[1],
                                        self.objects[0]['name'],
                                        self.objects[0]['meta'])
        self.assertEqual(o, self.objects[0]['data'])
    
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
    def test_objects_with_trailing_spaces(self):
        self.client.create_container('test')
        #create 'a' object
        self.upload_random_data('test', 'a')
        #look for 'a ' object
        self.assert_raises_fault(404, self.client.retrieve_object,
                                 'test', 'a ')
        
        #delete 'a' object
        self.client.delete_object('test', 'a')
        self.assert_raises_fault(404, self.client.retrieve_object,
                                 'test', 'a')
        
        #create 'a ' object
        self.upload_random_data('test', 'a ')
        #look for 'a' object
        self.assert_raises_fault(404, self.client.retrieve_object,
                                 'test', 'a')
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
876
    def test_get_invalid(self):
877 878 879
        self.assert_raises_fault(404, self.client.retrieve_object,
                                 self.containers[0], self.objects[0]['name'])
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
880 881
    def test_get_partial(self):
        #perform get with range
882 883 884
        status, headers, data = self.client.request_object(self.containers[1],
                                                            self.objects[0]['name'],
                                                            range='bytes=0-499')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
885 886
        
        #assert successful partial content
887
        self.assertEqual(status, 206)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
888
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
889
        #assert content-type
890
        self.assertEqual(headers['content-type'],
891
                         self.objects[0]['meta']['content_type'])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
892
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
893
        #assert content length
894
        self.assertEqual(int(headers['content-length']), 500)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
895 896
        
        #assert content
897
        self.assertEqual(self.objects[0]['data'][:500], data)
898
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
899 900
    def test_get_final_500(self):
        #perform get with range
901
        headers = {'range':'bytes=-500'}
902 903 904
        status, headers, data = self.client.request_object(self.containers[1],
                                                            self.objects[0]['name'],
                                                            range='bytes=-500')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
905 906
        
        #assert successful partial content
907
        self.assertEqual(status, 206)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
908
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
909
        #assert content-type
910
        self.assertEqual(headers['content-type'],
911
                         self.objects[0]['meta']['content_type'])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
912
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
913
        #assert content length
914
        self.assertEqual(int(headers['content-length']), 500)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
915 916
        
        #assert content
917
        self.assertTrue(self.objects[0]['data'][-500:], data)
918
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
919 920 921
    def test_get_rest(self):
        #perform get with range
        offset = len(self.objects[0]['data']) - 500
922
        status, headers, data = self.client.request_object(self.containers[1],
923
                                                self.objects[0]['name'],
924
                                                range='bytes=%s-' %offset)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
925 926
        
        #assert successful partial content
927
        self.assertEqual(status, 206)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
928
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
929
        #assert content-type
930
        self.assertEqual(headers['content-type'],
931
                         self.objects[0]['meta']['content_type'])
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
932
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
933
        #assert content length
934
        self.assertEqual(int(headers['content-length']), 500)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
935 936
        
        #assert content
937
        self.assertTrue(self.objects[0]['data'][-500:], data)
938
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
939 940 941 942
    def test_get_range_not_satisfiable(self):
        #perform get with range
        offset = len(self.objects[0]['data']) + 1
        
943
        #assert range not satisfiable
944 945 946 947
        self.assert_raises_fault(416, self.client.retrieve_object,
                                 self.containers[1], self.objects[0]['name'],
                                 range='bytes=0-%s' %offset)
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
948 949 950
    def test_multiple_range(self):
        #perform get with multiple range
        ranges = ['0-499', '-500', '1000-']
951 952 953 954
        bytes = 'bytes=%s' % ','.join(ranges)
        status, headers, data = self.client.request_object(self.containers[1],
                                                           self.objects[0]['name'],
                                                           range=bytes)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed