Commit c3d131ec authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Merge branch 'feature-store' into develop

parents d7beb9ae 745d938b
......@@ -11,65 +11,62 @@ This guide describes the standard installation process for kamaki, with the aspi
Linux and Unix-like enviroments
-------------------------------
Ubuntu and Debian packages
^^^^^^^^^^^^^^^^^^^^^^^^^^
Debian:
^^^^^^^
The following steps describe a command-line approach, but any graphic package manager can be used instead.
Add the following to apt sources list
"""""""""""""""""""""""""""""""""""""
* As root, append the following to */etc/apt/sources.list* ::
As root, append one of the following to */etc/apt/sources.list*:
deb http://apt.dev.grnet.gr/ squeeze main
deb http://apt2.dev.grnet.gr stable/
* Make sure the GPG public key for the GRNET dev team is added:
* Debian Stable::
.. code-block:: console
deb http://apt.dev.grnet.gr/ squeeze main
deb http://apt2.dev.grnet.gr stable/
$ sudo curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
* Ubuntu::
otherwise *apt-get update* will produce GPG warnings.
deb http://ppa.launchpad.net/grnet/synnefo/ubuntu <UBUNTU_VERSION> main
deb-src http://ppa.launchpad.net/grnet/synnefo/ubuntu <UBUNTU_VERSION> main
* Update the Debian sources:
where <UBUNTU_VERSION> is the system Ubuntu version (e.g. precise)
.. code-block:: console
Update
""""""
$ sudo apt-get update
* In Debian, make sure the GPG public key for the GRNET dev team is added:
* Install kamaki:
.. code-block:: console
$ curl https://dev.grnet.gr/files/apt-grnetdev.pub|apt-key add -
$ sudo apt-get install kamaki
otherwise *apt-get update* will produce GPG warnings.
Ubuntu
^^^^^^
The following steps describe a command-line approach, but any graphic package manager can be used instead.
* In Ubuntu, make sure the correct PPA signing key is installed:
* Let ppa take care of the repository configuration:
.. code-block:: console
$ sudo apt-get install python-software-properties
$ sudo add-apt-repository ppa:grnet/synnefo
Finally:
.. code-block:: console
$ sudo apt-get update
* Update the Debian sources:
.. code-block:: console
Install kamaki
""""""""""""""
$ sudo apt-get update
.. note:: The objpool package is available at synnefo repositories and is automatically installed as a dependency.
* Install kamaki:
.. code-block:: console
.. code-block:: console
$ sudo apt-get install kamaki
$ sudo apt-get install kamaki
Install ansicolors and/or progress (Optional)
"""""""""""""""""""""""""""""""""""""""""""""
Install ansicolors and/or progress (Optional but recommended)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
.. code-block:: console
......
This diff is collapsed.
......@@ -175,14 +175,12 @@ class PithosClient(PithosRestAPI):
def _put_block_async(self, data, hash, upload_gen=None):
event = SilentEvent(method=self._put_block, data=data, hash=hash)
event.start()
if upload_gen:
upload_gen.next()
return event
def _put_block(self, data, hash):
#from random import randint
#if randint(0,2):
# raise ClientError('BAD GATEWAY STUFF', 502)
from random import randint
if not randint(0, 7):
raise ClientError('BAD GATEWAY STUFF', 503)
r = self.container_post(update=True,
content_type='application/octet-stream',
content_length=len(data),
......@@ -253,14 +251,8 @@ class PithosClient(PithosRestAPI):
"Failed to calculate uploaded blocks: " \
"Offset and object size do not match"
def _upload_missing_blocks(self, missing, hmap, fileobj, upload_cb=None):
"""upload missing blocks asynchronously.
"""
if upload_cb:
upload_gen = upload_cb(len(missing))
upload_gen.next()
else:
upload_gen = None
def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None):
"""upload missing blocks asynchronously"""
self._init_thread_limit()
......@@ -272,24 +264,31 @@ class PithosClient(PithosRestAPI):
data = fileobj.read(bytes)
r = self._put_block_async(data, hash, upload_gen)
flying.append(r)
unfinished = []
for thread in flying:
unfinished = self._watch_thread_limit(unfinished)
unfinished = self._watch_thread_limit(flying)
for thread in set(flying).difference(unfinished):
if thread.exception:
failures.append(thread)
if isinstance(thread.exception, ClientError)\
and thread.exception.status == 502:
self.POOLSIZE = self._thread_limit
elif thread.isAlive():
unfinished.append(thread)
flying.append(thread)
elif upload_gen:
try:
upload_gen.next()
except:
pass
flying = unfinished
for thread in flying:
thread.join()
if thread.exception:
failures.append(thread)
elif upload_gen:
try:
upload_gen.next()
except:
pass
return [failure.kwargs['hash'] for failure in failures]
......@@ -354,15 +353,26 @@ class PithosClient(PithosRestAPI):
if missing is None:
return
retries = 3
if upload_cb:
upload_gen = upload_cb(len(missing))
for i in range(len(missing), len(hashmap['hashes']) + 1):
try:
upload_gen.next()
except:
upload_gen = None
else:
upload_gen = None
retries = 7
try:
while retries:
sendlog.info('%s blocks missing' % len(missing))
num_of_blocks = len(missing)
missing = self._upload_missing_blocks(
missing,
hmap,
f,
upload_cb=upload_cb)
upload_gen)
if missing:
if num_of_blocks == len(missing):
retries -= 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment