dummy.py 13.2 KB
Newer Older
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
1
2
import os
import sqlite3
3
import logging
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
4
import types
5
import hashlib
Antony Chazapis's avatar
Antony Chazapis committed
6
import shutil
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
7
import basebackend
8
9

logger = logging.getLogger(__name__)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
10

Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
11
class BackEnd(basebackend.BaseBackEnd):
12

Antony Chazapis's avatar
Antony Chazapis committed
13
    def __init__(self, basepath):
14
        self.basepath = basepath
15
        
16
17
        if not os.path.exists(basepath):
            os.makedirs(basepath)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
18
        db = os.path.join(basepath, 'db')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
19
        self.con = sqlite3.connect(db)
20
21
        # Create tables
        sql = '''create table if not exists objects(name text)'''
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
22
        self.con.execute(sql)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
23
24
        sql = '''create table if not exists metadata(object_id int, name text, value text)'''
        self.con.execute(sql)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
25
        self.con.commit()
26
27
    
    # TODO: Create/delete account?
Antony Chazapis's avatar
Antony Chazapis committed
28
    # TODO: Catch OSError exceptions.
29
    
30
    def get_account_meta(self, account):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
31
        """
32
        returns a dictionary with the account metadata
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
33
        """
34
        logger.debug("get_account_meta: %s", account)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
35
        fullname = os.path.join(self.basepath, account)
36
37
        if not os.path.exists(fullname):
            raise NameError('Account does not exist')
38
        contents = os.listdir(fullname)
39
        count = len(contents)
40
        size = os.stat(fullname).st_size
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
        meta = self.__get_metadata(account)
        meta.update({'name': account, 'count': count, 'bytes': size})
        return meta

    def update_account_meta(self, account, meta):
        """
        updates the metadata associated with the account
        """
        logger.debug("update_account_meta: %s %s", account, meta)
        fullname = os.path.join(self.basepath, account)
        if not os.path.exists(fullname):
            os.makedirs(fullname)
        self.__put_metadata(account, meta)
        return
    
56
    def create_container(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
57
58
        """
        creates a new container with the given name
59
        if it doesn't exist under the basepath
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
60
        """
61
        logger.debug("create_container: %s %s", account, name)
62
        fullname = os.path.join(self.basepath, account, name)
63
        if not os.path.exists(fullname):
64
            os.makedirs(fullname)
65
66
67
        else:
            raise NameError('Container already exists')
        return
68
    
69
    def delete_container(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
70
71
        """
        deletes the container with the given name
72
        if it exists under the basepath and is empty
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
73
        """
74
        logger.debug("delete_container: %s %s", account, name)
75
        fullname = os.path.join(self.basepath, account, name)
76
77
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
78
        if os.listdir(fullname):
Antony Chazapis's avatar
Antony Chazapis committed
79
            raise IndexError('Container is not empty')
80
        else:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
81
            os.rmdir(fullname)
82
            self.__del_dbpath(os.path.join(account, name))
83
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
84
    
85
    def get_container_meta(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
86
87
88
        """
        returns a dictionary with the container metadata
        """
89
        logger.debug("get_container_meta: %s %s", account, name)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
90
        fullname = os.path.join(self.basepath, account, name)
91
92
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
93
        contents = os.listdir(fullname)
94
        count = len(contents)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
95
        size = os.stat(fullname).st_size
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        meta = self.__get_metadata(os.path.join(account, name))
        meta.update({'name': name, 'count': count, 'bytes': size})
        return meta
    
    def update_container_meta(self, account, name, meta):
        """
        updates the metadata associated with the container
        """
        logger.debug("update_container_meta: %s %s %s", account, name, meta)
        fullname = os.path.join(self.basepath, account, name)
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
        self.__put_metadata(os.path.join(account, name), meta)
        return
110
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
111
112
    def list_containers(self, account, marker = None, limit = 10000):
        """
113
114
        returns a list of at most limit (default = 10000) containers 
        starting from the next item after the optional marker
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
115
        """
116
        logger.debug("list_containers: %s %s %s", account, marker, limit)
117
118
119
        fullname = os.path.join(self.basepath, account)
        if not os.path.exists(fullname):
            raise NameError('Account does not exist')
120
        containers = os.listdir(fullname)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
121
122
123
        start = 0
        if marker:
            try:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
124
                start = containers.index(marker) + 1
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
125
126
            except ValueError:
                pass
Antony Chazapis's avatar
Antony Chazapis committed
127
128
129
130
        if not limit or limit > 10000:
            limit = 10000
        
        return containers[start:start + limit]
131
    
Antony Chazapis's avatar
Antony Chazapis committed
132
    def list_objects(self, account, container, prefix = '', delimiter = None, marker = None, limit = 10000):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
133
        """
Antony Chazapis's avatar
Antony Chazapis committed
134
        returns a list of objects existing under a container
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
135
        """
136
        logger.info("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
Antony Chazapis's avatar
Antony Chazapis committed
137
138
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
139
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
140
141
142
143
144
145
146
        
        while prefix.startswith('/'):
            prefix = prefix[1:]
        # TODO: Test this with various prefixes. Does '//' bother it?
        prefix = os.path.join(account, container, prefix)
        c = self.con.execute('select * from objects where name like ''?'' order by name', (os.path.join(prefix, '%'),))
        objects = [x[0][len(prefix):] for x in c.fetchall()]
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
147
        if delimiter:
Antony Chazapis's avatar
Antony Chazapis committed
148
            pseudo_objects = []
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
149
            for x in objects:
Antony Chazapis's avatar
Antony Chazapis committed
150
                pseudo_name = x
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
151
152
153
                i = pseudo_name.find(delimiter)
                if i != -1:
                    pseudo_name = pseudo_name[:i]
Antony Chazapis's avatar
Antony Chazapis committed
154
155
                if pseudo_name not in pseudo_objects:
                    pseudo_objects.append(pseudo_name)
Antony Chazapis's avatar
Antony Chazapis committed
156
                # TODO: Virtual directories.
Antony Chazapis's avatar
Antony Chazapis committed
157
            objects = pseudo_objects
Antony Chazapis's avatar
Antony Chazapis committed
158
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
159
160
161
        start = 0
        if marker:
            try:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
162
                start = objects.index(marker) + 1
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
163
164
165
166
            except ValueError:
                pass
        if not limit or limit > 10000:
            limit = 10000
Antony Chazapis's avatar
Antony Chazapis committed
167
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
168
        return objects[start:start + limit]
169
    
Antony Chazapis's avatar
Antony Chazapis committed
170
171
172
173
174
175
176
    def get_object_meta(self, account, container, name, keys = None):
        """
        returns a dictionary with the object metadata
        """
        logger.info("get_object_meta: %s %s %s %s", account, container, name, keys)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
177
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
178
179
180
181
182
183
184
185
186
187
188
        
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        size = os.path.getsize(location)
        mtime = os.path.getmtime(location)
        meta = self.__get_metadata(os.path.join(account, container, name))
        meta.update({'name': name, 'bytes': size, 'last_modified': mtime})
        if 'hash' not in meta:
            meta['hash'] = self.__object_hash(location)
        if 'content_type' not in meta:
            meta['content_type'] = 'application/octet-stream'
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
189
        return meta
190
    
Antony Chazapis's avatar
Antony Chazapis committed
191
192
193
194
195
196
197
    def update_object_meta(self, account, container, name, meta):
        """
        updates the metadata associated with the object
        """
        logger.info("update_object_meta: %s %s %s %s", account, container, name, meta)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
198
            raise NameError('Container does not exist')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
199
200
201
202
        try:
            link = self.__get_linkinfo(os.path.join(account, container, name))
        except NameError:
            raise NameError('Object does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
203
204
205
206
207
208
209
210
211
212
213
214
215
216
        self.__put_metadata(os.path.join(account, container, name), meta)
        return
    
    def get_object(self, account, container, name, offset = 0, length = -1):
        """
        returns the object data
        """
        logger.info("get_object: %s %s %s %s %s", account, container, name, offset, length)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
        
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
217
218
219
220
221
222
        f = open(location, 'r')
        if offset:
            f.seek(offset)
        data = f.read(length)
        f.close()
        return data
Antony Chazapis's avatar
Antony Chazapis committed
223
224
225
226
227
228
229
230

    def update_object(self, account, container, name, data, offset = 0):
        """
        creates/updates an object with the specified data
        """
        logger.info("put_object: %s %s %s %s %s", account, container, name, data, offset)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
231
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
232

233
        try:
Antony Chazapis's avatar
Antony Chazapis committed
234
            link = self.__get_linkinfo(os.path.join(account, container, name))
235
236
        except NameError:
            # new object
Antony Chazapis's avatar
Antony Chazapis committed
237
238
239
240
241
242
243
244
            link = self.__put_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        f = open(location, 'w')
        if offset:
            f.seek(offset)
        f.write(data)
        f.close()
        self.__put_metadata(os.path.join(account, container, name), {'hash': self.__object_hash(location)})
245
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
246
    
Antony Chazapis's avatar
Antony Chazapis committed
247
248
249
250
251
252
253
254
255
256
257
    def copy_object(self, account, src_container, src_name, dest_container, dest_name):
        """
        copies an object
        """
        logger.info("copy_object: %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name)
        link = self.__get_linkinfo(os.path.join(account, src_container, src_name))
        src_location = os.path.join(self.basepath, account, src_container, link)
        
        dest_fullname = os.path.join(self.basepath, account, dest_container)
        if not os.path.exists(dest_fullname):
            raise NameError('Destination container does not exist')        
258
        try:
Antony Chazapis's avatar
Antony Chazapis committed
259
            link = self.__get_linkinfo(os.path.join(account, dest_container, dest_name))
260
261
        except NameError:
            # new object
Antony Chazapis's avatar
Antony Chazapis committed
262
263
264
265
            link = self.__put_linkinfo(os.path.join(account, dest_container, dest_name))
        dest_location = os.path.join(self.basepath, account, dest_container, link)
        
        shutil.copyfile(src_location, dest_location)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
266
267
        # TODO: accept metadata changes
        self.__put_metadata(os.path.join(account, dest_container, dest_name), self.__get_metadata(os.path.join(account, src_container, src_name)))
268
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
269
    
270
    def delete_object(self, account, container, name):
Antony Chazapis's avatar
Antony Chazapis committed
271
272
273
274
275
276
        """
        deletes an object
        """
        logger.info("delete_object: %s %s %s", account, container, name)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
277
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
278
        
279
        # delete object data
Antony Chazapis's avatar
Antony Chazapis committed
280
281
282
283
284
285
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        try:
            os.remove(location)
        except:
            pass
286
        # delete object metadata
Antony Chazapis's avatar
Antony Chazapis committed
287
        self.__del_dbpath(os.path.join(account, container, name))
288
        return
Antony Chazapis's avatar
Antony Chazapis committed
289
290
291
292
293
294
295
296

    def __get_linkinfo(self, path):
        c = self.con.execute('select rowid from objects where name=''?''', (path,))
        row = c.fetchone()
        if row:
            return str(row[0])
        else:
            raise NameError('Requested path not found')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
297
    
Antony Chazapis's avatar
Antony Chazapis committed
298
299
    def __put_linkinfo(self, path):
        id = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
300
        self.con.commit()
Antony Chazapis's avatar
Antony Chazapis committed
301
        return str(id)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
302
    
Antony Chazapis's avatar
Antony Chazapis committed
303
304
305
    def __get_metadata(self, path):
        c = self.con.execute('select m.name, m.value from metadata m, objects o where o.rowid = m.object_id and o.name = ''?''', (path,))
        return dict(c.fetchall())
306
    
Antony Chazapis's avatar
Antony Chazapis committed
307
308
    def __put_metadata(self, path, meta):
        c = self.con.execute('select rowid from objects where name=''?''', (path,))
309
310
        row = c.fetchone()
        if row:
Antony Chazapis's avatar
Antony Chazapis committed
311
            link = str(row[0])
312
        else:
Antony Chazapis's avatar
Antony Chazapis committed
313
314
315
            link = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid      
        for k, v in meta.iteritems():
            self.con.execute('insert or replace into metadata (object_id, name, value) values (?, ?, ?)', (link, k, v))
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
316
        self.con.commit()
Antony Chazapis's avatar
Antony Chazapis committed
317
318
319
320
321
322
323
        return

    def __del_dbpath(self, path):
        self.con.execute('delete from metadata where object_id in (select rowid from objects where name = ''?'')', (path,))
        self.con.execute('delete from objects where name = ''?''', (path,))
        self.con.commit()
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
324
    
Antony Chazapis's avatar
Antony Chazapis committed
325
326
327
328
329
330
331
332
333
334
    def __object_hash(self, location, block_size = 8192):
        md5 = hashlib.md5()
        f = open(location, 'r')
        while True:
            data = f.read(block_size)
            if not data:
                break
            md5.update(data)
        f.close()
        return md5.hexdigest()