Commit d2cea1e2 authored by Giorgos Verigakis's avatar Giorgos Verigakis
Browse files

Add Pithos support

* Use Pithos' smart upload if available
* Add store container command
* Add store delete command
parent a1c50326
......@@ -123,8 +123,15 @@ def command(api=None, group=None, name=None, description=None, syntax=None):
class config_list(object):
"""list configuration options"""
@classmethod
def update_parser(cls, parser):
parser.add_option('-a', dest='all', action='store_true',
default=False, help='include empty values')
def main(self):
for key, val in sorted(self.config.items()):
if not val and not self.options.all:
continue
print '%s=%s' % (key, val)
......@@ -634,6 +641,24 @@ class glance_setmembers(object):
self.client.set_members(image_id, member)
@command(api='storage')
class store_container(object):
"""get container info"""
@classmethod
def update_parser(cls, parser):
parser.add_option('--account', dest='account', metavar='ACCOUNT',
help='use account ACCOUNT')
parser.add_option('--container', dest='container', metavar='CONTAINER',
help='use container CONTAINER')
def main(self):
self.config.override('storage_account', self.options.account)
self.config.override('storage_container', self.options.container)
reply = self.client.get_container_meta()
print_dict(reply)
@command(api='storage')
class store_upload(object):
"""upload a file"""
......@@ -646,13 +671,34 @@ class store_upload(object):
help='use container CONTAINER')
def main(self, path, remote_path=None):
account = self.options.account or self.config.get('storage_account')
container = self.options.container or \
self.config.get('storage_container')
self.config.override('storage_account', self.options.account)
self.config.override('storage_container', self.options.container)
# Use the more efficient Pithos client if available
if 'pithos' in self.config.get('apis').split():
self.client = clients.PithosClient(self.config)
if remote_path is None:
remote_path = basename(path)
with open(path) as f:
self.client.create_object(account, container, remote_path, f)
self.client.create_object(remote_path, f)
@command(api='storage')
class store_delete(object):
"""delete a file"""
@classmethod
def update_parser(cls, parser):
parser.add_option('--account', dest='account', metavar='ACCOUNT',
help='use account ACCOUNT')
parser.add_option('--container', dest='container', metavar='CONTAINER',
help='use container CONTAINER')
def main(self, path):
self.config.override('storage_account', self.options.account)
self.config.override('storage_container', self.options.container)
self.client.delete_object(path)
def print_groups(groups):
......@@ -764,14 +810,11 @@ def main():
cmd.config = config
cmd.options = options
if cmd.api in ('compute', 'image', 'storage', 'cyclades'):
token = config.get('token')
if cmd.api in ('compute', 'image', 'storage'):
url = config.get(cmd.api + '_url')
elif cmd.api == 'cyclades':
url = config.get('compute_url')
cls_name = cmd.api.capitalize() + 'Client'
cmd.client = getattr(clients, cls_name)(url, token)
if cmd.api:
client_name = cmd.api.capitalize() + 'Client'
client = getattr(clients, client_name, None)
if client:
cmd.client = client(config)
try:
ret = cmd.main(*args[3:])
......
......@@ -53,3 +53,4 @@ from .compute import ComputeClient
from .image import ImageClient
from .storage import StorageClient
from .cyclades import CycladesClient
from .pithos import PithosClient
......@@ -31,16 +31,28 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""
OpenStack Compute API 1.1 client
"""
import json
from .http import HTTPClient
class ComputeClient(HTTPClient):
class ComputeClient(HTTPClient):
"""OpenStack Compute API 1.1 client"""
@property
def url(self):
url = self.config.get('compute_url') or self.config.get('url')
if not url:
raise ClientError('No URL was given')
return url
@property
def token(self):
token = self.config.get('compute_token') or self.config.get('token')
if not token:
raise ClientError('No token was given')
return token
def list_servers(self, detail=False):
"""List servers, returned detailed output if detailed is True"""
path = '/servers/detail' if detail else '/servers'
......
......@@ -31,16 +31,14 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""
GRNet Cyclades API client
"""
import json
from .http import HTTPClient
from .compute import ComputeClient
class CycladesClient(HTTPClient):
class CycladesClient(ComputeClient):
"""GRNet Cyclades API client"""
def start_server(self, server_id):
"""Submit a startup request for a server specified by id"""
path = '/servers/%d/action' % server_id
......
......@@ -44,9 +44,22 @@ log = logging.getLogger('kamaki.clients')
class HTTPClient(object):
def __init__(self, url, token):
self.url = url
self.token = token
def __init__(self, config):
self.config = config
@property
def url(self):
url = self.config.get('url')
if not url:
raise ClientError('No URL was given')
return url
@property
def token(self):
token = self.config.get('token')
if not token:
raise ClientError('No token was given')
return token
def raw_http_cmd(self, method, path, body=None, headers=None, success=200,
json_reply=False):
......@@ -62,7 +75,7 @@ class HTTPClient(object):
headers = headers or {}
headers['X-Auth-Token'] = self.token
if body:
headers['Content-Type'] = 'application/json'
headers.setdefault('Content-Type', 'application/json')
headers['Content-Length'] = len(body)
log.debug('>' * 50)
......@@ -93,7 +106,7 @@ class HTTPClient(object):
raise ClientError('Did not receive valid JSON reply',
resp.status, reply)
if resp.status != success:
if success and resp.status != success:
if len(reply) == 1:
if json_reply:
key = reply.keys()[0]
......
......@@ -41,6 +41,20 @@ from .http import HTTPClient
class ImageClient(HTTPClient):
@property
def url(self):
url = self.config.get('image_url') or self.config.get('url')
if not url:
raise ClientError('No URL was given')
return url
@property
def token(self):
token = self.config.get('image_token') or self.config.get('token')
if not token:
raise ClientError('No token was given')
return token
def list_public(self, detail=False, filters={}, order=''):
path = '/images/detail' if detail else '/images/'
params = {}
......
# Copyright 2011 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
import hashlib
import json
from . import ClientError
from .storage import StorageClient
class PithosClient(StorageClient):
"""GRNet Pithos API client"""
def put_block(self, data):
path = '/%s/%s?update' % (self.account, self.container)
headers = {'Content-Type': 'application/octet-stream'}
self.raw_http_cmd('POST', path, data, headers, success=202)
def create_object(self, object, f):
meta = self.get_container_meta()
blocksize = int(meta['block-size'])
blockhash = meta['block-hash']
size = 0
hashes = []
data = f.read(blocksize)
while data:
size += len(data)
h = hashlib.new(blockhash)
h.update(data)
hashes.append(h.hexdigest())
data = f.read(blocksize)
path = '/%s/%s/%s?hashmap&format=json' % (self.account, self.container,
object)
hashmap = dict(bytes=size, hashes=hashes)
req = json.dumps(hashmap)
resp, reply = self.raw_http_cmd('PUT', path, req, success=None)
if resp.status not in (201, 409):
raise ClientError('Invalid response from the server')
if resp.status == 201:
return
hashes = set(reply.split())
f.seek(0)
data = f.read(blocksize)
while data:
h = hashlib.new(blockhash)
h.update(data)
hash = h.hexdigest()
if hash in hashes:
self.put_block(data)
hashes.remove(hash)
data = f.read(blocksize)
self.http_put(path, req, success=201)
......@@ -31,19 +31,51 @@
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
"""
OpenStack Object Storage API 1.0 client
"""
from . import ClientError
from .http import HTTPClient
class StorageClient(HTTPClient):
def __init__(self, url, token, account, container):
self.url = url
self.token = token
self.account = account
self.container = container
"""OpenStack Object Storage API 1.0 client"""
@property
def url(self):
url = self.config.get('storage_url') or self.config.get('url')
if not url:
raise ClientError('No URL was given')
return url
@property
def token(self):
token = self.config.get('storage_token') or self.config.get('token')
if not token:
raise ClientError('No token was given')
return token
@property
def account(self):
account = self.config.get('storage_account')
if not account:
raise ClientError('No account was given')
return account
@property
def container(self):
container = self.config.get('storage_container')
if not container:
raise ClientError('No container was given')
return container
def get_container_meta(self):
path = '/%s/%s' % (self.account, self.container)
resp, reply = self.raw_http_cmd('HEAD', path, success=204)
reply = {}
prefix = 'x-container-'
for key, val in resp.getheaders():
key = key.lower()
if key.startswith(prefix):
reply[key[len(prefix):]] = val
return reply
def create_object(self, object, f):
path = '/%s/%s/%s' % (self.account, self.container, object)
......
......@@ -46,14 +46,17 @@ CONFIG_ENV = 'KAMAKI_CONFIG'
# The defaults also determine the allowed keys
CONFIG_DEFAULTS = {
'apis': 'compute image storage cyclades',
'apis': 'compute image storage cyclades pithos',
'token': '',
'url': '',
'compute_token': '',
'compute_url': 'https://okeanos.grnet.gr/api/v1',
'image_token': '',
'image_url': 'https://okeanos.grnet.gr/plankton',
'storage_url': 'https://plus.pithos.grnet.gr/v1',
'storage_account': '',
'storage_container': '',
'test_token': ''
'storage_token': '',
'storage_url': 'https://plus.pithos.grnet.gr/v1'
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment