Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
itminedu
synnefo
Commits
b7108deb
Commit
b7108deb
authored
Apr 28, 2011
by
Sofia Papagiannaki
Browse files
μετατροπή σε class, include accounts
Refs #447
parent
67fc7478
Changes
1
Hide whitespace changes
Inline
Side-by-side
pithos/backends/dummy.py
View file @
b7108deb
import
os
import
sqlite3
import
json
import
logging
basepath
=
'/Users/butters/src/pithos/backends/content'
#full path
if
not
os
.
path
.
exists
(
basepath
):
os
.
makedirs
(
basepath
)
db
=
'/'
.
join
([
basepath
,
'db'
])
con
=
sqlite3
.
connect
(
db
)
# Create tables
print
'Creating tables....'
sql
=
'''create table if not exists objects(name varchar(2560))'''
print
sql
con
.
execute
(
sql
)
# Save (commit) the changes
con
.
commit
()
def
create_container
(
name
):
""" creates a new container with the given name
if it doesn't exists under the basepath """
fullname
=
'/'
.
join
([
basepath
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
os
.
chdir
(
basepath
)
os
.
mkdir
(
name
)
else
:
raise
NameError
(
'Container already exists'
)
return
def
delete_container
(
name
):
""" deletes the container with the given name
if it exists under the basepath """
fullname
=
'/'
.
join
([
basepath
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Container does not exist'
)
if
not
list_objects
(
name
):
raise
Error
(
'Container is not empty'
)
else
:
os
.
chdir
(
basepath
)
os
.
rmdir
(
name
)
return
def
get_container_meta
(
name
):
""" returns a dictionary with the container metadata """
fullname
=
'/'
.
join
([
basepath
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Container does not exist'
)
contents
=
os
.
listdir
(
fullname
)
count
=
len
(
contents
)
size
=
sum
(
os
.
path
.
getsize
(
'/'
.
join
([
basepath
,
name
,
objectname
]))
for
objectname
in
contents
)
return
{
'name'
:
name
,
'count'
:
count
,
'bytes'
:
size
}
def
list_containers
():
return
os
.
listdir
(
basepath
)
def
list_objects
(
container
,
prefix
=
''
,
delimiter
=
None
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
search_str
=
''
if
prefix
:
search_str
=
'/'
.
join
([
search_str
,
prefix
])
#if delimiter:
if
None
:
search_str
=
''
.
join
([
'%'
,
search_str
,
'%'
,
delimiter
])
print
search_str
c
=
con
.
execute
(
'select * from objects where name like ''?'' order by name'
,
(
search_str
,))
else
:
search_str
=
''
.
join
([
'%'
,
search_str
,
'%'
])
print
search_str
c
=
con
.
execute
(
'select * from objects where name like ''?'' order by name'
,
(
search_str
,))
l
=
[]
for
row
in
c
.
fetchall
():
s
=
''
print
row
[
0
]
rest
=
str
(
row
[
0
]).
split
(
prefix
)[
1
]
print
rest
#if delimiter:
# rest = rest.partition(delimiter)[0]
#print rest
folders
=
rest
.
split
(
'/'
)[:
-
1
]
for
folder
in
folders
:
path
=
''
.
join
([
s
,
folder
,
'/'
])
if
path
not
in
l
:
l
.
append
(
path
)
s
=
''
.
join
([
s
,
folder
,
'/'
])
l
.
append
(
rest
)
return
l
def
get_object_meta
(
container
,
name
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
__get_object_linkinfo
(
'/'
.
join
([
container
,
name
]))
location
=
'.'
.
join
([
location
,
'meta'
])
f
=
open
(
location
,
'r'
)
data
=
json
.
load
(
f
)
f
.
close
()
return
data
def
get_object_data
(
container
,
name
,
offset
=
0
,
length
=-
1
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
__get_object_linkinfo
(
'/'
.
join
([
container
,
name
]))
f
=
open
(
location
,
'r'
)
if
offset
:
f
.
seek
(
offset
)
data
=
f
.
read
(
length
)
f
.
close
()
return
data
def
update_object
(
container
,
name
,
data
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
try
:
location
=
__get_object_linkinfo
(
'/'
.
join
([
container
,
name
]))
except
NameError
:
# new object
location
=
str
(
__save_linkinfo
(
'/'
.
join
([
container
,
name
])))
print
':'
.
join
([
'Creating new location'
,
location
])
__store_data
(
location
,
container
,
data
)
return
def
update_object_meta
(
container
,
name
,
meta
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
try
:
location
=
__get_object_linkinfo
(
'/'
.
join
([
container
,
name
]))
except
NameError
:
# new object
location
=
str
(
__save_linkinfo
(
'/'
.
join
([
container
,
name
])))
print
':'
.
join
([
'Creating new location'
,
location
])
__store_metadata
(
location
,
container
,
meta
)
return
class
BackEnd
:
def
__init__
(
self
,
basepath
,
log_file
=
'backend.out'
,
log_level
=
logging
.
DEBUG
):
self
.
basepath
=
basepath
logging
.
basicConfig
(
filename
=
log_file
,
level
=
log_level
,)
if
not
os
.
path
.
exists
(
basepath
):
os
.
makedirs
(
basepath
)
db
=
'/'
.
join
([
basepath
,
'db'
])
con
=
sqlite3
.
connect
(
db
)
# Create tables
sql
=
'''create table if not exists objects(name text)'''
con
.
execute
(
sql
)
con
.
commit
()
def
copy_object
(
src_container
,
src_name
,
dest_container
,
dest_name
,
meta
):
fullname
=
'/'
.
join
([
basepath
,
dest_container
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Destination container does not exist'
)
update_object
(
dest_container
,
dest_name
,
get_object_data
(
src_container
,
src_name
))
src_object_meta
=
get_object_meta
(
src_container
,
src_name
)
if
(
type
(
src_object_meta
)
==
types
.
DictType
):
distinct_keys
=
[
k
for
k
in
src_object_meta
.
keys
()
if
k
not
in
meta
.
keys
()]
for
k
in
distinct_keys
:
meta
[
k
]
=
src_object_meta
[
k
]
update_object_meta
(
dest_container
,
dest_name
,
meta
)
else
:
update_object_meta
(
dest_container
,
dest_name
,
meta
)
return
def
get_account_meta
(
self
,
account
):
""" returns a dictionary with the container metadata """
logging
.
info
(
"get_account_meta: %s %s"
,
account
)
fullname
=
'/'
.
join
([
self
.
basepath
,
account
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Account does not exist'
)
contents
=
os
.
listdir
(
fullname
)
count
=
len
(
contents
)
size
=
sum
(
os
.
path
.
getsize
(
'/'
.
join
([
self
.
basepath
,
account
,
objectname
]))
for
objectname
in
contents
)
return
{
'name'
:
name
,
'count'
:
count
,
'bytes'
:
size
}
def
create_container
(
self
,
account
,
name
):
""" creates a new container with the given name
if it doesn't exists under the basepath """
logging
.
debug
(
"create_container: %s %s"
,
account
,
name
)
fullname
=
'/'
.
join
([
self
.
basepath
,
account
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
os
.
chdir
(
self
.
basepath
)
os
.
mkdir
(
name
)
else
:
raise
NameError
(
'Container already exists'
)
return
def
delete_object
(
container
,
name
):
return
def
__store_metadata
(
location
,
container
,
meta
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
'.'
.
join
([
location
,
'meta'
])
f
=
open
(
location
,
'w'
)
data
=
json
.
dumps
(
meta
)
f
.
write
(
data
)
f
.
close
()
def
__store_data
(
location
,
container
,
data
):
dir
=
'/'
.
join
([
basepath
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
f
=
open
(
location
,
'w'
)
f
.
write
(
data
)
f
.
close
()
def
delete_container
(
self
,
account
,
name
):
""" deletes the container with the given name
if it exists under the basepath
and it's empty"""
logging
.
debug
(
"delete_container: %s %s"
,
params
.
items
()[
1
:])
fullname
=
'/'
.
join
([
self
.
basepath
,
account
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Container does not exist'
)
if
not
list_objects
(
name
):
raise
Error
(
'Container is not empty'
)
else
:
os
.
chdir
(
self
.
basepath
)
os
.
rmdir
(
name
)
return
def
__get_object_linkinfo
(
name
):
c
=
con
.
execute
(
'select rowid from objects where name=''?''', (name,))
row = c.fetchone()
if row:
return str(row[0])
else:
raise NameError('
Object
not
found
')
def __save_linkinfo(name):
id = con.execute('
insert
into
objects
(
name
)
values
(
?
)
', (name,)).lastrowid
con.commit()
return id
def
get_container_meta
(
self
,
account
,
name
):
""" returns a dictionary with the container metadata """
fullname
=
'/'
.
join
([
self
.
basepath
,
account
,
name
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Container does not exist'
)
contents
=
os
.
listdir
(
fullname
)
count
=
len
(
contents
)
size
=
sum
(
os
.
path
.
getsize
(
'/'
.
join
([
self
.
basepath
,
account
,
name
,
objectname
]))
for
objectname
in
contents
)
return
{
'name'
:
name
,
'count'
:
count
,
'bytes'
:
size
}
def
list_containers
(
self
,
marker
=
None
,
limit
=
10000
):
return
os
.
listdir
(
self
.
basepath
)[:
limit
]
def
list_objects
(
self
,
account
,
container
,
prefix
=
''
,
delimiter
=
None
,
marker
=
None
,
limit
=
10000
):
#dir = '/'.join([self.basepath, account, container])
#if not os.path.exists(dir):
# raise NameError('Container does not exist')
#search_str = ''
#if prefix:
# search_str = '/'.join([search_str, prefix])
##if delimiter:
#if None:
# search_str = ''.join(['%', search_str, '%', delimiter])
# print search_str
# c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
#else:
# search_str = ''.join(['%', search_str, '%'])
# print search_str
# c = con.execute('select * from objects where name like ''?'' order by name', (search_str,))
#l = []
#for row in c.fetchall():
# s = ''
# print row[0]
# rest = str(row[0]).split(prefix)[1]
# print rest
# #if delimiter:
# # rest = rest.partition(delimiter)[0]
# #print rest
# folders = rest.split('/')[:-1]
# for folder in folders:
# path = ''.join([s, folder, '/'])
# if path not in l:
# l.append(path)
# s = ''.join([s, folder, '/'])
# l.append(rest)
logging
.
info
(
"list_objects: %s %s %s %s %s %s"
,
account
,
container
,
prefix
,
delimiter
,
marker
,
limit
)
if
prefix
or
delimiter
:
if
prefix
:
objects
=
[
x
for
x
in
objects
if
x
[
'name'
].
startswith
(
prefix
)]
if
delimiter
:
pseudo_objects
=
{}
for
x
in
objects
:
pseudo_name
=
x
[
'name'
][
len
(
prefix
):]
i
=
pseudo_name
.
find
(
delimiter
)
if
i
!=
-
1
:
pseudo_name
=
pseudo_name
[:
i
]
# TODO: Virtual directories.
if
pseudo_name
not
in
pseudo_objects
:
pseudo_objects
[
pseudo_name
]
=
x
objects
=
sorted
(
pseudo_objects
.
values
(),
key
=
lambda
o
:
o
[
'name'
])
start
=
0
if
marker
:
try
:
start
=
binary_search_name
(
objects
,
marker
)
+
1
except
ValueError
:
pass
if
not
limit
or
limit
>
10000
:
limit
=
10000
return
objects
[
start
:
start
+
limit
]
def
get_object_meta
(
self
,
account
,
container
,
name
,
keys
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
self
.
__get_object_linkinfo
(
'/'
.
join
([
account
,
container
,
name
]))
location
=
'.'
.
join
([
location
,
'meta'
])
f
=
open
(
location
,
'r'
)
data
=
json
.
load
(
f
)
f
.
close
()
return
data
def
get_object_data
(
self
,
account
,
container
,
name
,
offset
=
0
,
length
=-
1
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
self
.
__get_object_linkinfo
(
'/'
.
join
([
account
,
container
,
name
]))
f
=
open
(
location
,
'r'
)
if
offset
:
f
.
seek
(
offset
)
data
=
f
.
read
(
length
)
f
.
close
()
return
data
def
update_object
(
self
,
account
,
container
,
name
,
data
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
try
:
location
=
self
.
__get_object_linkinfo
(
'/'
.
join
([
account
,
container
,
name
]))
except
NameError
:
# new object
location
=
str
(
self
.
__save_linkinfo
(
'/'
.
join
([
account
,
container
,
name
])))
print
':'
.
join
([
'Creating new location'
,
location
])
self
.
__store_data
(
location
,
container
,
data
)
return
if __name__ == '
__main__
':
dirname = '
papagian
'
#create_container(dirname)
#assert os.path.exists(dirname)
#assert os.path.isdir(dirname)
def
update_object_meta
(
self
,
account
,
container
,
name
,
meta
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
try
:
location
=
self
.
__get_object_linkinfo
(
'/'
.
join
([
account
,
container
,
name
]))
except
NameError
:
# new object
location
=
str
(
self
.
__save_linkinfo
(
'/'
.
join
([
account
,
container
,
name
])))
print
':'
.
join
([
'Creating new location'
,
location
])
self
.
__store_metadata
(
location
,
container
,
meta
)
return
#print get_container_meta(dirname)
def
copy_object
(
self
,
account
,
src_container
,
src_name
,
dest_container
,
dest_name
,
meta
):
fullname
=
'/'
.
join
([
self
.
basepath
,
account
,
dest_container
])
if
not
os
.
path
.
exists
(
fullname
):
raise
NameError
(
'Destination container does not exist'
)
self
.
update_object
(
account
,
dest_container
,
dest_name
,
get_object_data
(
account
,
src_container
,
src_name
))
src_object_meta
=
self
.
get_object_meta
(
account
,
src_container
,
src_name
)
if
(
type
(
src_object_meta
)
==
types
.
DictType
):
distinct_keys
=
[
k
for
k
in
src_object_meta
.
keys
()
if
k
not
in
meta
.
keys
()]
for
k
in
distinct_keys
:
meta
[
k
]
=
src_object_meta
[
k
]
self
.
update_object_meta
(
account
,
dest_container
,
dest_name
,
meta
)
else
:
self
.
update_object_meta
(
account
,
dest_container
,
dest_name
,
meta
)
return
#update_object_meta(dirname, '
photos
/
animals
/
dog
.
jpg
', {'
name
':'
dog
.
jpg
'})
#update_object_meta(dirname, '
photos
/
animals
/
dog
.
jpg
', {'
name
':'
dog
.
jpg
', '
type
':'
image
', '
size
':400})
#print get_object_meta(dirname, '
photos
/
animals
/
dog
.
jpg
')
def
delete_object
(
self
,
account
,
container
,
name
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
self
.
__get_object_linkinfo
(
'/'
.
join
([
account
,
container
,
name
]))
# delete object data
self
.
__delete_data
(
location
,
account
,
container
)
# delete object metadata
location
=
'.'
.
join
([
location
,
'meta'
])
self
.
__delete_data
(
location
,
account
,
container
)
return
#f = open('
dummy
.
py
')
#data = f.read()
#update_object(dirname, '
photos
/
animals
/
dog
.
jpg
', data)
#update_object(dirname, '
photos
/
animals
/
cat
.
jpg
', data)
#update_object(dirname, '
photos
/
animals
/
thumbs
/
cat
.
jpg
', data)
#update_object(dirname, '
photos
/
fruits
/
banana
.
jpg
', data)
def
__store_metadata
(
self
,
location
,
account
,
container
,
meta
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
location
=
'.'
.
join
([
location
,
'meta'
])
f
=
open
(
location
,
'w'
)
data
=
json
.
dumps
(
meta
)
f
.
write
(
data
)
f
.
close
()
#print list_objects(dirname, '
photos
/
animals
');
def
__store_data
(
self
,
location
,
account
,
container
,
data
):
dir
=
'/'
.
join
([
self
.
basepath
,
account
,
container
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
chdir
(
dir
)
f
=
open
(
location
,
'w'
)
f
.
write
(
data
)
f
.
close
()
def
__delete_data
(
self
,
location
,
account
,
container
):
file
=
'/'
.
join
([
self
.
basepath
,
account
,
container
,
location
])
if
not
os
.
path
.
exists
(
dir
):
raise
NameError
(
'Container does not exist'
)
else
:
os
.
remove
(
file
)
def
__get_object_linkinfo
(
self
,
name
):
c
=
con
.
execute
(
'select rowid from objects where name=''?''', (name,))
row = c.fetchone()
if row:
return str(row[0])
else:
raise NameError('
Object
not
found
')
copy_object(dirname, '
photos
/
animals
/
dog
.
jpg
', '
photos
/
animals
/
dog2
.
jpg
')
copy_object(dirname, '
photos
/
animals
/
dg
.
jpg
', '
photos
/
animals
/
dog2
.
jpg
')
def __save_linkinfo(self, name):
id = con.execute('
insert
into
objects
(
name
)
values
(
?
)
', (name,)).lastrowid
con.commit()
return id
def __delete_linkinfo(self, name):
con.execute('
delete
from
objects
where
name
=
?
', (name,))
cont.commit()
return
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment