pithos.py 4.22 KB
Newer Older
Giorgos Verigakis's avatar
Giorgos Verigakis committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# Copyright 2011 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

import hashlib
Giorgos Verigakis's avatar
Giorgos Verigakis committed
35
import os
Giorgos Verigakis's avatar
Giorgos Verigakis committed
36

37
from ..utils import OrderedDict
Giorgos Verigakis's avatar
Giorgos Verigakis committed
38

Giorgos Verigakis's avatar
Giorgos Verigakis committed
39
40
41
42
43
44
45
46
from .storage import StorageClient


def pithos_hash(block, blockhash):
    h = hashlib.new(blockhash)
    h.update(block.rstrip('\x00'))
    return h.hexdigest()

Giorgos Verigakis's avatar
Giorgos Verigakis committed
47
48
49
50

class PithosClient(StorageClient):
    """GRNet Pithos API client"""
    
Giorgos Verigakis's avatar
Giorgos Verigakis committed
51
    def put_block(self, data, hash):
Giorgos Verigakis's avatar
Giorgos Verigakis committed
52
53
        path = '/%s/%s' % (self.account, self.container)
        params = {'update': ''}
Giorgos Verigakis's avatar
Giorgos Verigakis committed
54
        headers = {'Content-Type': 'application/octet-stream',
Giorgos Verigakis's avatar
Giorgos Verigakis committed
55
56
57
58
                   'Content-Length': str(len(data))}
        r = self.post(path, params=params, data=data, headers=headers,
                      success=202)
        assert r.text.strip() == hash, 'Local hash does not match server'
Giorgos Verigakis's avatar
Giorgos Verigakis committed
59
    
60
61
    def create_object(self, object, f, size=None, hash_cb=None,
                      upload_cb=None):
Giorgos Verigakis's avatar
Giorgos Verigakis committed
62
63
64
65
66
67
68
69
70
71
72
73
        """Create an object by uploading only the missing blocks
        
        hash_cb is a generator function taking the total number of blocks to
        be hashed as an argument. Its next() will be called every time a block
        is hashed.
        
        upload_cb is a generator function with the same properties that is
        called every time a block is uploaded.
        """
        self.assert_container()
        
        meta = self.get_container_meta(self.container)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
74
75
76
        blocksize = int(meta['block-size'])
        blockhash = meta['block-hash']
        
77
        file_size = size if size is not None else os.fstat(f.fileno()).st_size
Giorgos Verigakis's avatar
Giorgos Verigakis committed
78
        nblocks = 1 + (file_size - 1) // blocksize
79
        hashes = OrderedDict()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
80
81
82
83
84
85
86
87
88
89
        
        size = 0
        
        if hash_cb:
            hash_gen = hash_cb(nblocks)
            hash_gen.next()
        for i in range(nblocks):
            block = f.read(blocksize)
            bytes = len(block)
            hash = pithos_hash(block, blockhash)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
90
91
            hashes[hash] = (size, bytes)
            size += bytes
Giorgos Verigakis's avatar
Giorgos Verigakis committed
92
93
94
95
            if hash_cb:
                hash_gen.next()
        
        assert size == file_size
Giorgos Verigakis's avatar
Giorgos Verigakis committed
96
                
Giorgos Verigakis's avatar
Giorgos Verigakis committed
97
        path = '/%s/%s/%s' % (self.account, self.container, object)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
98
        params = dict(format='json', hashmap='')
99
        hashmap = dict(bytes=size, hashes=hashes.keys())
Giorgos Verigakis's avatar
Giorgos Verigakis committed
100
        r = self.put(path, params=params, json=hashmap, success=(201, 409))
Giorgos Verigakis's avatar
Giorgos Verigakis committed
101
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
102
        if r.status_code == 201:
Giorgos Verigakis's avatar
Giorgos Verigakis committed
103
104
            return
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
105
        missing = r.json
Giorgos Verigakis's avatar
Giorgos Verigakis committed
106
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
107
108
109
        if upload_cb:
            upload_gen = upload_cb(len(missing))
            upload_gen.next()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
110
111
112
113
114
        for hash in missing:
            offset, bytes = hashes[hash]
            f.seek(offset)
            data = f.read(bytes)
            self.put_block(data, hash)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
115
116
            if upload_cb:
                upload_gen.next()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
117
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
118
        self.put(path, params=params, json=hashmap, success=201)