dummy.py 13.2 KB
Newer Older
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
1
2
3
import os
import sqlite3
import json
4
import logging
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
5
import types
6
import hashlib
Antony Chazapis's avatar
Antony Chazapis committed
7
import shutil
8
9
10
11
12
13

logger = logging.getLogger(__name__)
formatter = logging.Formatter('[%(levelname)s] %(message)s')
handler = logging.FileHandler('backend.out')
handler.setFormatter(formatter)
logger.addHandler(handler)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
14

15
class BackEnd:
16
17
18

    logger = None
    
19
20
    def __init__(self, basepath, log_file='backend.out', log_level=logging.DEBUG):
        self.basepath = basepath
21
        
22
23
        # TODO: Manage log_file.
        logger.setLevel(log_level)
24
        
25
26
        if not os.path.exists(basepath):
            os.makedirs(basepath)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
27
        db = os.path.join(basepath, 'db')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
28
        self.con = sqlite3.connect(db)
29
30
        # Create tables
        sql = '''create table if not exists objects(name text)'''
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
31
        self.con.execute(sql)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
32
33
        sql = '''create table if not exists metadata(object_id int, name text, value text)'''
        self.con.execute(sql)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
34
        self.con.commit()
35
36
    
    # TODO: Create/delete account?
Antony Chazapis's avatar
Antony Chazapis committed
37
    # TODO: Catch OSError exceptions.
38
    
39
    def get_account_meta(self, account):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
40
        """
41
        returns a dictionary with the account metadata
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
42
        """
43
        logger.debug("get_account_meta: %s", account)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
44
        fullname = os.path.join(self.basepath, account)
45
46
        if not os.path.exists(fullname):
            raise NameError('Account does not exist')
47
        contents = os.listdir(fullname)
48
        count = len(contents)
49
        size = os.stat(fullname).st_size
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
        meta = self.__get_metadata(account)
        meta.update({'name': account, 'count': count, 'bytes': size})
        return meta

    def update_account_meta(self, account, meta):
        """
        updates the metadata associated with the account
        """
        logger.debug("update_account_meta: %s %s", account, meta)
        fullname = os.path.join(self.basepath, account)
        if not os.path.exists(fullname):
            os.makedirs(fullname)
        self.__put_metadata(account, meta)
        return
    
65
    def create_container(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
66
67
        """
        creates a new container with the given name
68
        if it doesn't exist under the basepath
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
69
        """
70
        logger.debug("create_container: %s %s", account, name)
71
        fullname = os.path.join(self.basepath, account, name)
72
        if not os.path.exists(fullname):
73
            os.makedirs(fullname)
74
75
76
        else:
            raise NameError('Container already exists')
        return
77
    
78
    def delete_container(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
79
80
        """
        deletes the container with the given name
81
        if it exists under the basepath and is empty
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
82
        """
83
        logger.debug("delete_container: %s %s", account, name)
84
        fullname = os.path.join(self.basepath, account, name)
85
86
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
87
88
        if os.listdir(fullname):
            raise Exception('Container is not empty')
89
        else:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
90
            os.rmdir(fullname)
91
            self.__del_dbpath(os.path.join(account, name))
92
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
93
    
94
    def get_container_meta(self, account, name):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
95
96
97
        """
        returns a dictionary with the container metadata
        """
98
        logger.debug("get_container_meta: %s %s", account, name)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
99
        fullname = os.path.join(self.basepath, account, name)
100
101
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
102
        contents = os.listdir(fullname)
103
        count = len(contents)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
104
        size = os.stat(fullname).st_size
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        meta = self.__get_metadata(os.path.join(account, name))
        meta.update({'name': name, 'count': count, 'bytes': size})
        return meta
    
    def update_container_meta(self, account, name, meta):
        """
        updates the metadata associated with the container
        """
        logger.debug("update_container_meta: %s %s %s", account, name, meta)
        fullname = os.path.join(self.basepath, account, name)
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
        self.__put_metadata(os.path.join(account, name), meta)
        return
119
    
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
120
121
    def list_containers(self, account, marker = None, limit = 10000):
        """
122
123
        returns a list of at most limit (default = 10000) containers 
        starting from the next item after the optional marker
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
124
        """
125
        logger.debug("list_containers: %s %s %s", account, marker, limit)
126
127
128
        fullname = os.path.join(self.basepath, account)
        if not os.path.exists(fullname):
            raise NameError('Account does not exist')
129
        containers = os.listdir(fullname)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
130
131
132
        start = 0
        if marker:
            try:
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
133
                start = containers.index(marker) + 1
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
134
135
            except ValueError:
                pass
Antony Chazapis's avatar
Antony Chazapis committed
136
137
138
139
        if not limit or limit > 10000:
            limit = 10000
        
        return containers[start:start + limit]
140
    
Antony Chazapis's avatar
Antony Chazapis committed
141
    def list_objects(self, account, container, prefix = '', delimiter = None, marker = None, limit = 10000):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
142
        """
Antony Chazapis's avatar
Antony Chazapis committed
143
        returns a list of objects existing under a container
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
144
        """
145
        logger.info("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
Antony Chazapis's avatar
Antony Chazapis committed
146
147
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
148
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
149
150
151
152
153
154
155
        
        while prefix.startswith('/'):
            prefix = prefix[1:]
        # TODO: Test this with various prefixes. Does '//' bother it?
        prefix = os.path.join(account, container, prefix)
        c = self.con.execute('select * from objects where name like ''?'' order by name', (os.path.join(prefix, '%'),))
        objects = [x[0][len(prefix):] for x in c.fetchall()]
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
156
157
158
        if delimiter:
            pseudo_objects = {}
            for x in objects:
Antony Chazapis's avatar
Antony Chazapis committed
159
                pseudo_name = x
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
160
161
162
                i = pseudo_name.find(delimiter)
                if i != -1:
                    pseudo_name = pseudo_name[:i]
Antony Chazapis's avatar
Antony Chazapis committed
163
                # TODO: Virtual directories.
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
164
165
                pseudo_objects[pseudo_name] = x
            objects = pseudo_objects.keys()
Antony Chazapis's avatar
Antony Chazapis committed
166
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
167
168
169
170
171
172
173
174
        start = 0
        if marker:
            try:
                start = objects.index(marker)
            except ValueError:
                pass
        if not limit or limit > 10000:
            limit = 10000
Antony Chazapis's avatar
Antony Chazapis committed
175
        
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
176
        return objects[start:start + limit]
177
    
Antony Chazapis's avatar
Antony Chazapis committed
178
179
180
181
182
183
184
    def get_object_meta(self, account, container, name, keys = None):
        """
        returns a dictionary with the object metadata
        """
        logger.info("get_object_meta: %s %s %s %s", account, container, name, keys)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
185
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
186
187
188
189
190
191
192
193
194
195
196
        
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        size = os.path.getsize(location)
        mtime = os.path.getmtime(location)
        meta = self.__get_metadata(os.path.join(account, container, name))
        meta.update({'name': name, 'bytes': size, 'last_modified': mtime})
        if 'hash' not in meta:
            meta['hash'] = self.__object_hash(location)
        if 'content_type' not in meta:
            meta['content_type'] = 'application/octet-stream'
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
197
        return meta
198
    
Antony Chazapis's avatar
Antony Chazapis committed
199
200
201
202
203
204
205
    def update_object_meta(self, account, container, name, meta):
        """
        updates the metadata associated with the object
        """
        logger.info("update_object_meta: %s %s %s %s", account, container, name, meta)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
206
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
        self.__put_metadata(os.path.join(account, container, name), meta)
        return
    
    def get_object(self, account, container, name, offset = 0, length = -1):
        """
        returns the object data
        """
        logger.info("get_object: %s %s %s %s %s", account, container, name, offset, length)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
            raise NameError('Container does not exist')
        
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
221
222
223
224
225
226
        f = open(location, 'r')
        if offset:
            f.seek(offset)
        data = f.read(length)
        f.close()
        return data
Antony Chazapis's avatar
Antony Chazapis committed
227
228
229
230
231
232
233
234

    def update_object(self, account, container, name, data, offset = 0):
        """
        creates/updates an object with the specified data
        """
        logger.info("put_object: %s %s %s %s %s", account, container, name, data, offset)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
235
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
236

237
        try:
Antony Chazapis's avatar
Antony Chazapis committed
238
            link = self.__get_linkinfo(os.path.join(account, container, name))
239
240
        except NameError:
            # new object
Antony Chazapis's avatar
Antony Chazapis committed
241
242
243
244
245
246
247
248
            link = self.__put_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        f = open(location, 'w')
        if offset:
            f.seek(offset)
        f.write(data)
        f.close()
        self.__put_metadata(os.path.join(account, container, name), {'hash': self.__object_hash(location)})
249
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
250
    
Antony Chazapis's avatar
Antony Chazapis committed
251
252
253
254
255
256
257
258
259
260
261
    def copy_object(self, account, src_container, src_name, dest_container, dest_name):
        """
        copies an object
        """
        logger.info("copy_object: %s %s %s %s %s", account, src_container, src_name, dest_container, dest_name)
        link = self.__get_linkinfo(os.path.join(account, src_container, src_name))
        src_location = os.path.join(self.basepath, account, src_container, link)
        
        dest_fullname = os.path.join(self.basepath, account, dest_container)
        if not os.path.exists(dest_fullname):
            raise NameError('Destination container does not exist')        
262
        try:
Antony Chazapis's avatar
Antony Chazapis committed
263
            link = self.__get_linkinfo(os.path.join(account, dest_container, dest_name))
264
265
        except NameError:
            # new object
Antony Chazapis's avatar
Antony Chazapis committed
266
267
268
269
            link = self.__put_linkinfo(os.path.join(account, dest_container, dest_name))
        dest_location = os.path.join(self.basepath, account, dest_container, link)
        
        shutil.copyfile(src_location, dest_location)
270
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
271
    
272
    def delete_object(self, account, container, name):
Antony Chazapis's avatar
Antony Chazapis committed
273
274
275
276
277
278
        """
        deletes an object
        """
        logger.info("delete_object: %s %s %s", account, container, name)
        fullname = os.path.join(self.basepath, account, container)
        if not os.path.exists(fullname):
279
            raise NameError('Container does not exist')
Antony Chazapis's avatar
Antony Chazapis committed
280
        
281
        # delete object data
Antony Chazapis's avatar
Antony Chazapis committed
282
283
284
285
286
287
        link = self.__get_linkinfo(os.path.join(account, container, name))
        location = os.path.join(self.basepath, account, container, link)
        try:
            os.remove(location)
        except:
            pass
288
        # delete object metadata
Antony Chazapis's avatar
Antony Chazapis committed
289
        self.__del_dbpath(os.path.join(account, container, name))
290
        return
Antony Chazapis's avatar
Antony Chazapis committed
291
292
293
294
295
296
297
298

    def __get_linkinfo(self, path):
        c = self.con.execute('select rowid from objects where name=''?''', (path,))
        row = c.fetchone()
        if row:
            return str(row[0])
        else:
            raise NameError('Requested path not found')
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
299
    
Antony Chazapis's avatar
Antony Chazapis committed
300
301
    def __put_linkinfo(self, path):
        id = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
302
        self.con.commit()
Antony Chazapis's avatar
Antony Chazapis committed
303
        return str(id)
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
304
    
Antony Chazapis's avatar
Antony Chazapis committed
305
306
307
    def __get_metadata(self, path):
        c = self.con.execute('select m.name, m.value from metadata m, objects o where o.rowid = m.object_id and o.name = ''?''', (path,))
        return dict(c.fetchall())
308
    
Antony Chazapis's avatar
Antony Chazapis committed
309
310
    def __put_metadata(self, path, meta):
        c = self.con.execute('select rowid from objects where name=''?''', (path,))
311
312
        row = c.fetchone()
        if row:
Antony Chazapis's avatar
Antony Chazapis committed
313
            link = str(row[0])
314
        else:
Antony Chazapis's avatar
Antony Chazapis committed
315
316
            link = self.con.execute('insert into objects (name) values (?)', (path,)).lastrowid      
        for k, v in meta.iteritems():
317
318
            if type(v) != types.StringType:
                v = json.dumps(v)
Antony Chazapis's avatar
Antony Chazapis committed
319
            self.con.execute('insert or replace into metadata (object_id, name, value) values (?, ?, ?)', (link, k, v))
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
320
        self.con.commit()
Antony Chazapis's avatar
Antony Chazapis committed
321
322
323
324
325
326
327
        return

    def __del_dbpath(self, path):
        self.con.execute('delete from metadata where object_id in (select rowid from objects where name = ''?'')', (path,))
        self.con.execute('delete from objects where name = ''?''', (path,))
        self.con.commit()
        return
Sofia Papagiannaki's avatar
Sofia Papagiannaki committed
328
    
Antony Chazapis's avatar
Antony Chazapis committed
329
330
331
332
333
334
335
336
337
338
    def __object_hash(self, location, block_size = 8192):
        md5 = hashlib.md5()
        f = open(location, 'r')
        while True:
            data = f.read(block_size)
            if not data:
                break
            md5.update(data)
        f.close()
        return md5.hexdigest()