Commit ccfe9b93 authored by Antony Chazapis's avatar Antony Chazapis

Restructure repository to make pithos one package.

Refs #1735
parent ab9b2d49
global-include */templates/* */fixtures/* */static/*
global-exclude */.DS_Store
exclude */settings.py
include */settings.py.dist
include pithos/settings.d/*
prune docs
prune tools
prune other
......@@ -44,3 +44,24 @@ Go to:
This server is useful during development, but should not be used for deployment.
To deploy Pithos using Apache, take a look at the Administrator Guide in docs.
Using the tools
---------------
In the pithos/tools directory you will find the following utilities:
pithos-sh Pithos shell
pithos-sync Pithos synchronization client
pithos-fs Pithos FUSE implementation
pithos-test Pithos server tests
Also, the pithos/lib directory contains a python library that can be used
to access Pithos and manage stored objects. All tools use the included lib.
Connection options can be set via environmental variables:
PITHOS_USER Login user
PITHOS_AUTH Login token
PITHOS_SERVER Pithos server (default: plus.pithos.grnet.gr)
PITHOS_API Pithos server path (default: v1)
PITHOS_SYNC_CONTAINER Container to sync with (default: pithos)
......@@ -33,10 +33,10 @@ Permissions
:members:
:undoc-members:
Hashfiler
~~~~~~~~~
Store
~~~~~
.. automodule:: pithos.backends.lib.hashfiler
.. automodule:: pithos.backends.lib.hashfiler.store
:show-inheritance:
:members:
:undoc-members:
Client Library
==============
.. automodule:: tools.lib.client
.. automodule:: pithos.lib.client
:show-inheritance:
:members:
:undoc-members:
......@@ -41,8 +41,9 @@ from sqlalchemy.sql import select
from pithos import settings
from pithos.backends.modular import ModularBackend
from lib.hashmap import HashMap
from lib.migrate import Migration, Cache
from pithos.lib.hashmap import HashMap
from migrate import Migration, Cache
import os
......
......@@ -46,10 +46,12 @@ from django.conf import settings
from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED
from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE
from lib.transfer import upload
from lib.hashmap import HashMap
from lib.client import Fault
from lib.migrate import Migration, Cache
from pithos.lib.transfer import upload
from pithos.lib.hashmap import HashMap
from pithos.lib.client import Fault
from migrate import Migration, Cache
from calendar import timegm
from decimal import Decimal
from collections import defaultdict
......@@ -311,4 +313,4 @@ if __name__ == "__main__":
ot.create_objects()
f.close()
\ No newline at end of file
......@@ -33,13 +33,13 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from lib.migrate import Migration
from sqlalchemy import Table
from sqlalchemy.sql import select
from pithos.im.models import User
from migrate import Migration
import base64
class UserMigration(Migration):
......@@ -102,4 +102,4 @@ class UserMigration(Migration):
if __name__ == "__main__":
db = ''
m = UserMigration(db)
m.execute()
\ No newline at end of file
m.execute()
# Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of Django nor the names of its contributors may be used
# to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
VERSION = (0, 8, 2, 'alpha', 0)
def get_version():
version = '%s.%s' % (VERSION[0], VERSION[1])
if VERSION[2]:
version = '%s.%s' % (version, VERSION[2])
if VERSION[3:] == ('alpha', 0):
version = '%s pre-alpha' % version
else:
if VERSION[3] != 'final':
version = '%s %s %s' % (version, VERSION[3], VERSION[4])
return version
......@@ -47,7 +47,8 @@ from django.utils.encoding import smart_unicode, smart_str
from django.core.files.uploadhandler import FileUploadHandler
from django.core.files.uploadedfile import UploadedFile
from pithos.api.compat import parse_http_date_safe, parse_http_date
from pithos.lib.compat import parse_http_date_safe, parse_http_date
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
RangeNotSatisfiable, ServiceUnavailable)
......
......@@ -35,11 +35,12 @@ import sys
import os
import time
import logging
import hashlib
import binascii
from base import NotAllowedError, QuotaError, BaseBackend
from pithos.lib.hashmap import HashMap
( CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED ) = range(3)
inf = float('inf')
......@@ -50,34 +51,6 @@ ULTIMATE_ANSWER = 42
logger = logging.getLogger(__name__)
class HashMap(list):
def __init__(self, blocksize, blockhash):
super(HashMap, self).__init__()
self.blocksize = blocksize
self.blockhash = blockhash
def _hash_raw(self, v):
h = hashlib.new(self.blockhash)
h.update(v)
return h.digest()
def hash(self):
if len(self) == 0:
return self._hash_raw('')
if len(self) == 1:
return self.__getitem__(0)
h = list(self)
s = 2
while s < len(h):
s = s * 2
h += [('\x00' * len(h[0]))] * (s - len(h))
while len(h) > 1:
h = [self._hash_raw(h[x] + h[x + 1]) for x in range(0, len(h), 2)]
return h[0]
def backend_method(func=None, autocommit=1):
if func is None:
def fn(func):
......
......@@ -7,5 +7,8 @@ except ImportError:
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
sys.exit(1)
if __name__ == "__main__":
def main():
execute_manager(settings)
if __name__ == "__main__":
main()
......@@ -48,3 +48,5 @@ conf = join(PROJECT_PATH, 'settings.local')
if exists(conf):
execfile(conf)
elif exists('/etc/pithos/settings.local'):
execfile('/etc/pithos/settings.local')
Tools in this dir depend on being able to import pithos.
You need to adjust PYTHONPATH accordingly for this to work.
e.g.
export PYTHONPATH=$HOME/pithos
......@@ -41,11 +41,10 @@ from stat import S_IFDIR, S_IFREG
from sys import argv
from time import time
from lib.compat import parse_http_date
from lib.client import OOS_Client, Fault
from lib.fuse import FUSE, FuseOSError, Operations
from lib.util import get_user, get_auth, get_server
from pithos.lib.compat import parse_http_date
from pithos.lib.client import OOS_Client, Fault
from pithos.lib.fuse import FUSE, FuseOSError, Operations
from pithos.lib.util import get_user, get_auth, get_server
epoch = int(time())
......
......@@ -38,9 +38,10 @@ from optparse import OptionParser
from os import environ
from sys import argv, exit, stdin, stdout
from datetime import datetime
from lib.client import Pithos_Client, Fault
from lib.util import get_user, get_auth, get_server, get_api
from lib.transfer import upload, download
from pithos.lib.client import Pithos_Client, Fault
from pithos.lib.util import get_user, get_auth, get_server, get_api
from pithos.lib.transfer import upload, download
import json
import logging
......
......@@ -41,10 +41,10 @@ from os.path import exists, expanduser, isdir, isfile, join, split
from shutil import copyfile
from time import time
from lib.transfer import download, upload
from lib.client import Pithos_Client, Fault
from lib.hashmap import merkle
from lib.util import get_user, get_auth, get_server
from pithos.lib.transfer import download, upload
from pithos.lib.client import Pithos_Client, Fault
from pithos.lib.hashmap import merkle
from pithos.lib.util import get_user, get_auth, get_server
DEFAULT_CONTAINER = 'pithos'
......
......@@ -34,8 +34,9 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from lib.client import Pithos_Client, Fault
from lib.util import get_user, get_auth, get_server, get_api
from pithos.lib.client import Pithos_Client, Fault
from pithos.lib.util import get_user, get_auth, get_server, get_api
from xml.dom import minidom
from StringIO import StringIO
from hashlib import new as newhasher
......
......@@ -2,4 +2,3 @@
all_files = 1
build-dir = docs/build
source-dir = docs/source
......@@ -3,17 +3,34 @@
import os
from setuptools import setup, find_packages
from pithos import get_version
VERSION = os.popen("git describe --abbrev=0 --tags").read().strip(' \nv')
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
VERSION = get_version().replace(' ', '')
INSTALL_REQUIRES = [
'Django==1.2.3',
'South==0.7',
'httplib2==0.6.0',
'SQLAlchemy==0.6.3',
'MySQL-python==1.2.2',
'psycopg2==2.2.1'
]
setup(
name='Pithos',
version=VERSION,
description='Pithos file storage service',
description='Pithos file storage service and tools',
long_description=read('README'),
author='GRNET',
author_email='pithos@grnet.gr',
url='http://code.grnet.gr/projects/pithos',
packages=find_packages(),
#install_requires = INSTALL_REQUIRES,
license='BSD',
classifiers=[
'Development Status :: 3 - Alpha',
......@@ -21,5 +38,14 @@ setup(
'Programming Language :: Python',
'Topic :: Utilities',
'License :: OSI Approved :: BSD License',
],
entry_points={
'console_scripts': ['pithos-manage = pithos.manage:main']
},
scripts=[
'pithos/tools/pithos-sh',
'pithos/tools/pithos-sync',
'pithos/tools/pithos-fs',
'pithos/tools/pithos-test'
]
)
exclude lib/migrate*
README
======
Pithos is a file storage service, built by GRNET.
Learn more about Pithos at: http://code.grnet.gr/projects/pithos
Here you will find the following tools:
pithos-sh Pithos shell
pithos-sync Pithos synchronization client
pithos-fs Pithos FUSE implementation
Also, the lib folder contains a python library that can be
used to access Pithos and manage stored objects.
All tools use the included lib.
Connection options can be set via environmental variables:
PITHOS_USER Login user
PITHOS_AUTH Login token
PITHOS_SERVER Pithos server (default: plus.pithos.grnet.gr)
PITHOS_API Pithos server path (default: v1)
PITHOS_SYNC_CONTAINER Container to sync with (default: pithos)
# Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of Django nor the names of its contributors may be used
# to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import re
import datetime
import calendar
MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
__D = r'(?P<day>\d{2})'
__D2 = r'(?P<day>[ \d]\d)'
__M = r'(?P<mon>\w{3})'
__Y = r'(?P<year>\d{4})'
__Y2 = r'(?P<year>\d{2})'
__T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
def parse_http_date(date):
"""
Parses a date format as specified by HTTP RFC2616 section 3.3.1.
The three formats allowed by the RFC are accepted, even if only the first
one is still in widespread use.
Returns an floating point number expressed in seconds since the epoch, in
UTC.
"""
# emails.Util.parsedate does the job for RFC1123 dates; unfortunately
# RFC2616 makes it mandatory to support RFC850 dates too. So we roll
# our own RFC-compliant parsing.
for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
m = regex.match(date)
if m is not None:
break
else:
raise ValueError("%r is not in a valid HTTP date format" % date)
try:
year = int(m.group('year'))
if year < 100:
if year < 70:
year += 2000
else:
year += 1900
month = MONTHS.index(m.group('mon').lower()) + 1
day = int(m.group('day'))
hour = int(m.group('hour'))
min = int(m.group('min'))
sec = int(m.group('sec'))
result = datetime.datetime(year, month, day, hour, min, sec)
return calendar.timegm(result.utctimetuple())
except Exception:
raise ValueError("%r is not a valid date" % date)
def parse_http_date_safe(date):
"""
Same as parse_http_date, but returns None if the input is invalid.
"""
try:
return parse_http_date(date)
except Exception:
pass
#!/usr/bin/env python
import os
from setuptools import setup, find_packages
VERSION = os.popen("git describe --abbrev=0 --tags").read().strip(' \nv')
setup(
name='Pithos Tools',
version=VERSION,
description='Pithos file storage service tools',
author='GRNET',
author_email='pithos@grnet.gr',
url='http://code.grnet.gr/projects/pithos',
scripts=['pithos-sh', 'pithos-sync', 'pithos-fs'],
packages=['lib'],
license='BSD',
classifiers=[
'Development Status :: 3 - Alpha',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Topic :: Utilities',
'License :: OSI Approved :: BSD License',
]
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment