Commit b7108deb authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

μετατροπή σε class, include accounts

Refs #447
parent 67fc7478
import os
import sqlite3
import json
import logging
basepath = '/Users/butters/src/pithos/backends/content' #full path
if not os.path.exists(basepath):
os.makedirs(basepath)
db = '/'.join([basepath, 'db'])
con = sqlite3.connect(db)
# Create tables
print 'Creating tables....'
sql = '''create table if not exists objects(name varchar(2560))'''
print sql
con.execute(sql)
# Save (commit) the changes
con.commit()
def create_container(name):
""" creates a new container with the given name
if it doesn't exists under the basepath """
fullname = '/'.join([basepath, name])
if not os.path.exists(fullname):
os.chdir(basepath)
os.mkdir(name)
else:
raise NameError('Container already exists')
return
def delete_container(name):
""" deletes the container with the given name
if it exists under the basepath """
fullname = '/'.join([basepath, name])
if not os.path.exists(fullname):
raise NameError('Container does not exist')
if not list_objects(name):
raise Error('Container is not empty')
else:
os.chdir(basepath)
os.rmdir(name)
return
def get_container_meta(name):
""" returns a dictionary with the container metadata """
fullname = '/'.join([basepath, name])
if not os.path.exists(fullname):
raise NameError('Container does not exist')
contents = os.listdir(fullname)
count = len(contents)
size = sum(os.path.getsize('/'.join([basepath, name, objectname])) for objectname in contents)
return {'name': name, 'count': count, 'bytes': size}
def list_containers():
return os.listdir(basepath)
def list_objects(container, prefix='', delimiter=None):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
search_str = ''
if prefix:
search_str = '/'.join([search_str, prefix])
#if delimiter:
if None:
search_str = ''.join(['%', search_str, '%', delimiter])
print search_str
c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
else:
search_str = ''.join(['%', search_str, '%'])
print search_str
c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
l = []
for row in c.fetchall():
s = ''
print row[0]
rest = str(row[0]).split(prefix)[1]
print rest
#if delimiter:
# rest = rest.partition(delimiter)[0]
#print rest
folders = rest.split('/')[:-1]
for folder in folders:
path = ''.join([s, folder, '/'])
if path not in l:
l.append(path)
s = ''.join([s, folder, '/'])
l.append(rest)
return l
def get_object_meta(container, name):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = __get_object_linkinfo('/'.join([container, name]))
location = '.'.join([location, 'meta'])
f = open(location, 'r')
data = json.load(f)
f.close()
return data
def get_object_data(container, name, offset=0, length=-1):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = __get_object_linkinfo('/'.join([container, name]))
f = open(location, 'r')
if offset:
f.seek(offset)
data = f.read(length)
f.close()
return data
def update_object(container, name, data):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
try:
location = __get_object_linkinfo('/'.join([container, name]))
except NameError:
# new object
location = str(__save_linkinfo('/'.join([container, name])))
print ':'.join(['Creating new location', location])
__store_data(location, container, data)
return
def update_object_meta(container, name, meta):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
try:
location = __get_object_linkinfo('/'.join([container, name]))
except NameError:
# new object
location = str(__save_linkinfo('/'.join([container, name])))
print ':'.join(['Creating new location', location])
__store_metadata(location, container, meta)
return
class BackEnd:
def __init__(self, basepath, log_file='backend.out', log_level=logging.DEBUG):
self.basepath = basepath
logging.basicConfig(filename=log_file,level=log_level,)
if not os.path.exists(basepath):
os.makedirs(basepath)
db = '/'.join([basepath, 'db'])
con = sqlite3.connect(db)
# Create tables
sql = '''create table if not exists objects(name text)'''
con.execute(sql)
con.commit()
def copy_object(src_container, src_name, dest_container, dest_name, meta):
fullname = '/'.join([basepath, dest_container])
if not os.path.exists(fullname):
raise NameError('Destination container does not exist')
update_object(dest_container, dest_name, get_object_data(src_container, src_name))
src_object_meta = get_object_meta(src_container, src_name)
if (type(src_object_meta) == types.DictType):
distinct_keys = [k for k in src_object_meta.keys() if k not in meta.keys()]
for k in distinct_keys:
meta[k] = src_object_meta[k]
update_object_meta(dest_container, dest_name, meta)
else:
update_object_meta(dest_container, dest_name, meta)
return
def get_account_meta(self, account):
""" returns a dictionary with the container metadata """
logging.info("get_account_meta: %s %s", account)
fullname = '/'.join([self.basepath, account])
if not os.path.exists(fullname):
raise NameError('Account does not exist')
contents = os.listdir(fullname)
count = len(contents)
size = sum(os.path.getsize('/'.join([self.basepath, account, objectname])) for objectname in contents)
return {'name': name, 'count': count, 'bytes': size}
def create_container(self, account, name):
""" creates a new container with the given name
if it doesn't exists under the basepath """
logging.debug("create_container: %s %s", account, name)
fullname = '/'.join([self.basepath, account, name])
if not os.path.exists(fullname):
os.chdir(self.basepath)
os.mkdir(name)
else:
raise NameError('Container already exists')
return
def delete_object(container, name):
return
def __store_metadata(location, container, meta):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = '.'.join([location, 'meta'])
f = open(location, 'w')
data = json.dumps(meta)
f.write(data)
f.close()
def __store_data(location, container, data):
dir = '/'.join([basepath, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
f = open(location, 'w')
f.write(data)
f.close()
def delete_container(self, account, name):
""" deletes the container with the given name
if it exists under the basepath
and it's empty"""
logging.debug("delete_container: %s %s", params.items()[1:])
fullname = '/'.join([self.basepath, account, name])
if not os.path.exists(fullname):
raise NameError('Container does not exist')
if not list_objects(name):
raise Error('Container is not empty')
else:
os.chdir(self.basepath)
os.rmdir(name)
return
def __get_object_linkinfo(name):
c = con.execute('select rowid from objects where name=''?''', (name,))
row = c.fetchone()
if row:
return str(row[0])
else:
raise NameError('Object not found')
def __save_linkinfo(name):
id = con.execute('insert into objects(name) values(?)', (name,)).lastrowid
con.commit()
return id
def get_container_meta(self, account, name):
""" returns a dictionary with the container metadata """
fullname = '/'.join([self.basepath, account, name])
if not os.path.exists(fullname):
raise NameError('Container does not exist')
contents = os.listdir(fullname)
count = len(contents)
size = sum(os.path.getsize('/'.join([self.basepath, account, name, objectname])) for objectname in contents)
return {'name': name, 'count': count, 'bytes': size}
def list_containers(self, marker = None, limit = 10000):
return os.listdir(self.basepath)[:limit]
def list_objects(self, account, container, prefix='', delimiter=None, marker = None, limit = 10000):
#dir = '/'.join([self.basepath, account, container])
#if not os.path.exists(dir):
# raise NameError('Container does not exist')
#search_str = ''
#if prefix:
# search_str = '/'.join([search_str, prefix])
##if delimiter:
#if None:
# search_str = ''.join(['%', search_str, '%', delimiter])
# print search_str
# c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
#else:
# search_str = ''.join(['%', search_str, '%'])
# print search_str
# c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
#l = []
#for row in c.fetchall():
# s = ''
# print row[0]
# rest = str(row[0]).split(prefix)[1]
# print rest
# #if delimiter:
# # rest = rest.partition(delimiter)[0]
# #print rest
# folders = rest.split('/')[:-1]
# for folder in folders:
# path = ''.join([s, folder, '/'])
# if path not in l:
# l.append(path)
# s = ''.join([s, folder, '/'])
# l.append(rest)
logging.info("list_objects: %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit)
if prefix or delimiter:
if prefix:
objects = [x for x in objects if x['name'].startswith(prefix)]
if delimiter:
pseudo_objects = {}
for x in objects:
pseudo_name = x['name'][len(prefix):]
i = pseudo_name.find(delimiter)
if i != -1:
pseudo_name = pseudo_name[:i]
# TODO: Virtual directories.
if pseudo_name not in pseudo_objects:
pseudo_objects[pseudo_name] = x
objects = sorted(pseudo_objects.values(), key=lambda o: o['name'])
start = 0
if marker:
try:
start = binary_search_name(objects, marker) + 1
except ValueError:
pass
if not limit or limit > 10000:
limit = 10000
return objects[start:start + limit]
def get_object_meta(self, account, container, name, keys):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = self.__get_object_linkinfo('/'.join([account, container, name]))
location = '.'.join([location, 'meta'])
f = open(location, 'r')
data = json.load(f)
f.close()
return data
def get_object_data(self, account, container, name, offset=0, length=-1):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = self.__get_object_linkinfo('/'.join([account, container, name]))
f = open(location, 'r')
if offset:
f.seek(offset)
data = f.read(length)
f.close()
return data
def update_object(self, account, container, name, data):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
try:
location = self.__get_object_linkinfo('/'.join([account, container, name]))
except NameError:
# new object
location = str(self.__save_linkinfo('/'.join([account, container, name])))
print ':'.join(['Creating new location', location])
self.__store_data(location, container, data)
return
if __name__ == '__main__':
dirname = 'papagian'
#create_container(dirname)
#assert os.path.exists(dirname)
#assert os.path.isdir(dirname)
def update_object_meta(self, account, container, name, meta):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
try:
location = self.__get_object_linkinfo('/'.join([account, container, name]))
except NameError:
# new object
location = str(self.__save_linkinfo('/'.join([account, container, name])))
print ':'.join(['Creating new location', location])
self.__store_metadata(location, container, meta)
return
#print get_container_meta(dirname)
def copy_object(self, account, src_container, src_name, dest_container, dest_name, meta):
fullname = '/'.join([self.basepath, account, dest_container])
if not os.path.exists(fullname):
raise NameError('Destination container does not exist')
self.update_object(account, dest_container, dest_name, get_object_data(account, src_container, src_name))
src_object_meta = self.get_object_meta(account, src_container, src_name)
if (type(src_object_meta) == types.DictType):
distinct_keys = [k for k in src_object_meta.keys() if k not in meta.keys()]
for k in distinct_keys:
meta[k] = src_object_meta[k]
self.update_object_meta(account, dest_container, dest_name, meta)
else:
self.update_object_meta(account, dest_container, dest_name, meta)
return
#update_object_meta(dirname, 'photos/animals/dog.jpg', {'name':'dog.jpg'})
#update_object_meta(dirname, 'photos/animals/dog.jpg', {'name':'dog.jpg', 'type':'image', 'size':400})
#print get_object_meta(dirname, 'photos/animals/dog.jpg')
def delete_object(self, account, container, name):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = self.__get_object_linkinfo('/'.join([account, container, name]))
# delete object data
self.__delete_data(location, account, container)
# delete object metadata
location = '.'.join([location, 'meta'])
self.__delete_data(location, account, container)
return
#f = open('dummy.py')
#data = f.read()
#update_object(dirname, 'photos/animals/dog.jpg', data)
#update_object(dirname, 'photos/animals/cat.jpg', data)
#update_object(dirname, 'photos/animals/thumbs/cat.jpg', data)
#update_object(dirname, 'photos/fruits/banana.jpg', data)
def __store_metadata(self, location, account, container, meta):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
location = '.'.join([location, 'meta'])
f = open(location, 'w')
data = json.dumps(meta)
f.write(data)
f.close()
#print list_objects(dirname, 'photos/animals');
def __store_data(self, location, account, container, data):
dir = '/'.join([self.basepath, account, container])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.chdir(dir)
f = open(location, 'w')
f.write(data)
f.close()
def __delete_data(self, location, account, container):
file = '/'.join([self.basepath, account, container, location])
if not os.path.exists(dir):
raise NameError('Container does not exist')
else:
os.remove(file)
def __get_object_linkinfo(self, name):
c = con.execute('select rowid from objects where name=''?''', (name,))
row = c.fetchone()
if row:
return str(row[0])
else:
raise NameError('Object not found')
copy_object(dirname, 'photos/animals/dog.jpg', 'photos/animals/dog2.jpg')
copy_object(dirname, 'photos/animals/dg.jpg', 'photos/animals/dog2.jpg')
def __save_linkinfo(self, name):
id = con.execute('insert into objects(name) values(?)', (name,)).lastrowid
con.commit()
return id
def __delete_linkinfo(self, name):
con.execute('delete from objects where name = ?', (name,))
cont.commit()
return
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment