Commit e9195e9d authored by Stavros Sachtouris's avatar Stavros Sachtouris
Browse files

Remove fstat from clients.utils.readall

Refs: #4871

Fstat was used to compute the actual file size in order to
distinguish if an empty read was because the file ended.
It now considers an empty read to be an end of file.

This allows reading non-regular files.
parent 567a88c2
...@@ -315,17 +315,19 @@ class PithosClient(PithosRestClient): ...@@ -315,17 +315,19 @@ class PithosClient(PithosRestClient):
hash_gen = hash_cb(nblocks) hash_gen = hash_cb(nblocks)
hash_gen.next() hash_gen.next()
for i in range(nblocks): for i in xrange(nblocks):
block = readall(fileobj, min(blocksize, size - offset)) block = readall(fileobj, min(blocksize, size - offset))
bytes = len(block) bytes = len(block)
if bytes <= 0:
break
hash = _pithos_hash(block, blockhash) hash = _pithos_hash(block, blockhash)
hashes.append(hash) hashes.append(hash)
hmap[hash] = (offset, bytes) hmap[hash] = (offset, bytes)
offset += bytes offset += bytes
if hash_cb: if hash_cb:
hash_gen.next() hash_gen.next()
msg = ('Failed to calculate uploaded blocks:' msg = ('Failed to calculate uploading blocks: '
' Offset and object size do not match') 'read bytes(%s) != requested size (%s)' % (offset, size))
assert offset == size, msg assert offset == size, msg
def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None): def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None):
......
...@@ -94,15 +94,14 @@ def path4url(*args): ...@@ -94,15 +94,14 @@ def path4url(*args):
def readall(openfile, size, retries=7): def readall(openfile, size, retries=7):
"""Read a file until size is reached""" """Read a file until size is reached"""
from os import fstat
actual_size = fstat(openfile.fileno()).st_size - openfile.tell()
size = actual_size if actual_size < size else size
remains = size if size > 0 else 0 remains = size if size > 0 else 0
buf = '' buf = ''
for i in range(retries): for i in range(retries):
buf += openfile.read(remains) tmp_buf = openfile.read(remains)
remains = size - len(buf) if tmp_buf:
if remains: buf += tmp_buf
continue remains -= len(tmp_buf)
if remains > 0:
continue
return buf return buf
raise IOError('Failed to read %s bytes from file' % size) raise IOError('Failed to read %s bytes from file' % size)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment