pithos.py 4.57 KB
Newer Older
Giorgos Verigakis's avatar
Giorgos Verigakis committed
1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
Giorgos Verigakis's avatar
Giorgos Verigakis committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

import hashlib
Giorgos Verigakis's avatar
Giorgos Verigakis committed
35
import os
Giorgos Verigakis's avatar
Giorgos Verigakis committed
36

Giorgos Verigakis's avatar
Giorgos Verigakis committed
37
38
from time import time

Giorgos Verigakis's avatar
Giorgos Verigakis committed
39
40
41
42
43
44
45
46
from .storage import StorageClient


def pithos_hash(block, blockhash):
    h = hashlib.new(blockhash)
    h.update(block.rstrip('\x00'))
    return h.hexdigest()

Giorgos Verigakis's avatar
Giorgos Verigakis committed
47
48
49

class PithosClient(StorageClient):
    """GRNet Pithos API client"""
Giorgos Verigakis's avatar
Giorgos Verigakis committed
50
51
52
53
54
55
56
57

    def purge_container(self, container):
        self.assert_account()

        path = '/%s/%s' % (self.account, container)
        params = {'until': int(time())}
        self.delete(path, params=params, success=204)

Giorgos Verigakis's avatar
Giorgos Verigakis committed
58
    def put_block(self, data, hash):
Giorgos Verigakis's avatar
Giorgos Verigakis committed
59
60
        path = '/%s/%s' % (self.account, self.container)
        params = {'update': ''}
Giorgos Verigakis's avatar
Giorgos Verigakis committed
61
        headers = {'Content-Type': 'application/octet-stream',
Giorgos Verigakis's avatar
Giorgos Verigakis committed
62
63
64
65
                   'Content-Length': str(len(data))}
        r = self.post(path, params=params, data=data, headers=headers,
                      success=202)
        assert r.text.strip() == hash, 'Local hash does not match server'
Giorgos Verigakis's avatar
Giorgos Verigakis committed
66
    
67
68
    def create_object(self, object, f, size=None, hash_cb=None,
                      upload_cb=None):
Giorgos Verigakis's avatar
Giorgos Verigakis committed
69
70
71
72
73
74
75
76
77
78
79
80
        """Create an object by uploading only the missing blocks
        
        hash_cb is a generator function taking the total number of blocks to
        be hashed as an argument. Its next() will be called every time a block
        is hashed.
        
        upload_cb is a generator function with the same properties that is
        called every time a block is uploaded.
        """
        self.assert_container()
        
        meta = self.get_container_meta(self.container)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
81
82
83
        blocksize = int(meta['block-size'])
        blockhash = meta['block-hash']
        
84
        file_size = size if size is not None else os.fstat(f.fileno()).st_size
Giorgos Verigakis's avatar
Giorgos Verigakis committed
85
        nblocks = 1 + (file_size - 1) // blocksize
Giorgos Verigakis's avatar
Giorgos Verigakis committed
86
87
88
        hashes = []
        map = {}

Giorgos Verigakis's avatar
Giorgos Verigakis committed
89
90
91
92
93
        size = 0
        
        if hash_cb:
            hash_gen = hash_cb(nblocks)
            hash_gen.next()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
94
    
Giorgos Verigakis's avatar
Giorgos Verigakis committed
95
96
97
98
        for i in range(nblocks):
            block = f.read(blocksize)
            bytes = len(block)
            hash = pithos_hash(block, blockhash)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
99
100
            hashes.append(hash)
            map[hash] = (size, bytes)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
101
            size += bytes
Giorgos Verigakis's avatar
Giorgos Verigakis committed
102
103
104
105
            if hash_cb:
                hash_gen.next()
        
        assert size == file_size
Giorgos Verigakis's avatar
Giorgos Verigakis committed
106
                
Giorgos Verigakis's avatar
Giorgos Verigakis committed
107
        path = '/%s/%s/%s' % (self.account, self.container, object)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
108
        params = dict(format='json', hashmap='')
Giorgos Verigakis's avatar
Giorgos Verigakis committed
109
        hashmap = dict(bytes=size, hashes=hashes)
110
111
112
113
        headers = {'Content-Type': 'application/octet-stream'}
        r = self.put(path, params=params, headers=headers, json=hashmap,
                     success=(201, 409))

Giorgos Verigakis's avatar
Giorgos Verigakis committed
114
        if r.status_code == 201:
Giorgos Verigakis's avatar
Giorgos Verigakis committed
115
116
            return
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
117
        missing = r.json
Giorgos Verigakis's avatar
Giorgos Verigakis committed
118
        
Giorgos Verigakis's avatar
Giorgos Verigakis committed
119
120
121
        if upload_cb:
            upload_gen = upload_cb(len(missing))
            upload_gen.next()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
122

Giorgos Verigakis's avatar
Giorgos Verigakis committed
123
        for hash in missing:
Giorgos Verigakis's avatar
Giorgos Verigakis committed
124
            offset, bytes = map[hash]
Giorgos Verigakis's avatar
Giorgos Verigakis committed
125
126
127
            f.seek(offset)
            data = f.read(bytes)
            self.put_block(data, hash)
Giorgos Verigakis's avatar
Giorgos Verigakis committed
128
129
            if upload_cb:
                upload_gen.next()
Giorgos Verigakis's avatar
Giorgos Verigakis committed
130

Giorgos Verigakis's avatar
Giorgos Verigakis committed
131
132
        self.put(path, params=params, headers=headers, json=hashmap,
                 success=201)