pithos.py 68.4 KB
Newer Older
1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.command

34
from time import localtime, strftime
35
from io import StringIO
36
from pydoc import pager
37
from os import path, walk, makedirs
38

39
from kamaki.clients.pithos import PithosClient, ClientError
40

Stavros Sachtouris's avatar
Stavros Sachtouris committed
41
from kamaki.cli import command
42
from kamaki.cli.command_tree import CommandTree
43
from kamaki.cli.commands import (
44
45
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
    _name_filter, _optional_output_cmd)
46
from kamaki.cli.errors import (
47
48
    CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
    CLISyntaxError)
49
from kamaki.cli.argument import (
Stavros Sachtouris's avatar
Stavros Sachtouris committed
50
    FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
51
    ProgressBarArgument, RepeatableArgument, DataSizeArgument)
52
from kamaki.cli.utils import (
53
    format_size, bold, get_path_size, guess_mime_type)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
54

55
56
57
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
container_cmds = CommandTree(
    'container', 'Pithos+/Storage container level API commands')
58
59
60
sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
_commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
Stavros Sachtouris's avatar
Stavros Sachtouris committed
61

62

63
class _pithos_init(_command_init):
64
65
66
67
    """Initilize a pithos+ client
    There is always a default account (current user uuid)
    There is always a default container (pithos)
    """
68

69
70
    @DontRaiseKeyError
    def _custom_container(self):
71
        return self.config.get_cloud(self.cloud, 'pithos_container')
72
73
74

    @DontRaiseKeyError
    def _custom_uuid(self):
75
        return self.config.get_cloud(self.cloud, 'pithos_uuid')
76
77
78
79
80

    def _set_account(self):
        self.account = self._custom_uuid()
        if self.account:
            return
81
82
83
        astakos = getattr(self, 'auth_base', None)
        if astakos:
            self.account = astakos.user_term('id', self.token)
84
        else:
85
            raise CLIBaseUrlError(service='astakos')
86

87
    @errors.generic.all
88
    @addLogSettings
89
    def _run(self):
90
91
        cloud = getattr(self, 'cloud', None)
        if cloud:
92
93
94
95
            self.base_url = self._custom_url('pithos')
        else:
            self.cloud = 'default'
        self.token = self._custom_token('pithos')
96
        self.container = self._custom_container() or 'pithos'
97

98
99
100
        astakos = getattr(self, 'auth_base', None)
        if astakos:
            self.token = self.token or astakos.token
101
            if not self.base_url:
102
                pithos_endpoints = astakos.get_service_endpoints(
103
104
105
                    self._custom_type('pithos') or 'object-store',
                    self._custom_version('pithos') or '')
                self.base_url = pithos_endpoints['publicURL']
106
107
        else:
            raise CLIBaseUrlError(service='astakos')
108

109
        self._set_account()
110
        self.client = PithosClient(
111
            self.base_url, self.token, self.account, self.container)
112

113
114
115
    def main(self):
        self._run()

Stavros Sachtouris's avatar
Stavros Sachtouris committed
116

117
118
class _pithos_account(_pithos_init):
    """Setup account"""
119

120
    def __init__(self, arguments={}, auth_base=None, cloud=None):
121
        super(_pithos_account, self).__init__(arguments, auth_base, cloud)
122
        self['account'] = ValueArgument(
123
            'Use (a different) user uuid', ('-A', '--account'))
124

125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    def print_objects(self, object_list):
        for index, obj in enumerate(object_list):
            pretty_obj = obj.copy()
            index += 1
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
            if 'subdir' in obj:
                continue
            if self._is_dir(obj):
                size = 'D'
            else:
                size = format_size(obj['bytes'])
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
            oname = obj['name'] if self['more'] else bold(obj['name'])
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
            if self['detail']:
                self.writeln('%s%s' % (prfx, oname))
                self.print_dict(pretty_obj, exclude=('name'))
                self.writeln()
            else:
                oname = '%s%9s %s' % (prfx, size, oname)
                oname += '/' if self._is_dir(obj) else u''
                self.writeln(oname)
147

148
149
150
151
    @staticmethod
    def _is_dir(remote_dict):
        return 'application/directory' == remote_dict.get(
            'content_type', remote_dict.get('content-type', ''))
152

153
154
155
156
    def _run(self):
        super(_pithos_account, self)._run()
        self.client.account = self['account'] or getattr(
            self, 'account', getattr(self.client, 'account', None))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
157

158

159
160
class _pithos_container(_pithos_account):
    """Setup container"""
161

162
    def __init__(self, arguments={}, auth_base=None, cloud=None):
163
        super(_pithos_container, self).__init__(arguments, auth_base, cloud)
164
        self['container'] = ValueArgument(
165
            'Use this container (default: pithos)', ('-C', '--container'))
166

167
168
    @staticmethod
    def _resolve_pithos_url(url):
169
170
171
        """Match urls of one of the following formats:
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
        /CONTAINER/OBJECT_PATH
172
        return account, container, path
173
        """
174
        account, container, obj_path, prefix = '', '', url, 'pithos://'
175
        if url.startswith(prefix):
176
            account, sep, url = url[len(prefix):].partition('/')
177
178
            url = '/%s' % url
        if url.startswith('/'):
179
180
            container, sep, obj_path = url[1:].partition('/')
        return account, container, obj_path
181

182
    def _run(self, url=None):
183
        acc, con, self.path = self._resolve_pithos_url(url or '')
184
        #  self.account = acc or getattr(self, 'account', '')
185
        super(_pithos_container, self)._run()
186
        self.container = con or self['container'] or getattr(
187
            self, 'container', None) or getattr(self.client, 'container', '')
188
        self.client.account = acc or self.client.account
189
        self.client.container = self.container
190

191

192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
@command(file_cmds)
class file_info(_pithos_container, _optional_json):
    """Get information/details about a file"""

    arguments = dict(
        object_version=ValueArgument(
            'download a file of a specific version', '--object-version'),
        hashmap=FlagArgument(
            'Get file hashmap instead of details', '--hashmap'),
        matching_etag=ValueArgument(
            'show output if ETags match', '--if-match'),
        non_matching_etag=ValueArgument(
            'show output if ETags DO NOT match', '--if-none-match'),
        modified_since_date=DateArgument(
            'show output modified since then', '--if-modified-since'),
        unmodified_since_date=DateArgument(
            'show output unmodified since then', '--if-unmodified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
209
210
211
212
213
        sharing=FlagArgument(
            'show object permissions and sharing information', '--sharing'),
        metadata=FlagArgument('show only object metadata', '--metadata'),
        versions=FlagArgument(
            'show the list of versions for the file', '--object-versions')
214
215
    )

Stavros Sachtouris's avatar
Stavros Sachtouris committed
216
217
218
219
220
    def version_print(self, versions):
        return {'/%s/%s' % (self.container, self.path): [
            dict(version_id=vitem[0], created=strftime(
                '%d-%m-%Y %H:%M:%S',
                localtime(float(vitem[1])))) for vitem in versions]}
221

222
    @errors.generic.all
223
224
225
226
227
228
229
230
231
232
233
234
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self):
        if self['hashmap']:
            r = self.client.get_object_hashmap(
                self.path,
                version=self['object_version'],
                if_match=self['matching_etag'],
                if_none_match=self['non_matching_etag'],
                if_modified_since=self['modified_since_date'],
                if_unmodified_since=self['unmodified_since_date'])
Stavros Sachtouris's avatar
Stavros Sachtouris committed
235
        elif self['sharing']:
236
            r = self.client.get_object_sharing(self.path)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
237
238
239
240
241
242
243
244
245
246
            r['public url'] = self.client.get_object_info(
                self.path, version=self['object_version']).get(
                    'x-object-public', None)
        elif self['metadata']:
            r, preflen = dict(), len('x-object-meta-')
            for k, v in self.client.get_object_meta(self.path).items():
                r[k[preflen:]] = v
        elif self['versions']:
            r = self.version_print(
                self.client.get_object_versionlist(self.path))
247
248
249
250
251
252
253
254
255
256
        else:
            r = self.client.get_object_info(
                self.path, version=self['object_version'])
        self._print(r, self.print_dict)

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()


257
258
259
@command(file_cmds)
class file_list(_pithos_container, _optional_json, _name_filter):
    """List all objects in a container or a directory object"""
260

Stavros Sachtouris's avatar
Stavros Sachtouris committed
261
    arguments = dict(
262
263
264
        detail=FlagArgument('detailed output', ('-l', '--list')),
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
        marker=ValueArgument('output greater that marker', '--marker'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
265
266
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
        meta=ValueArgument(
267
            'show output with specified meta keys', '--meta',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
268
269
            default=[]),
        if_modified_since=ValueArgument(
270
            'show output modified since then', '--if-modified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
271
        if_unmodified_since=ValueArgument(
272
            'show output not modified since then', '--if-unmodified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
273
274
        until=DateArgument('show metadata until then', '--until'),
        format=ValueArgument(
275
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
276
277
278
        shared_by_me=FlagArgument(
            'show only files shared to other users', '--shared-by-me'),
        public=FlagArgument('show only published objects', '--public'),
279
        more=FlagArgument('read long results', '--more'),
280
281
282
283
        enum=FlagArgument('Enumerate results', '--enumerate'),
        recursive=FlagArgument(
            'Recursively list containers and their contents',
            ('-R', '--recursive'))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
284
    )
285

286
287
288
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
289
    @errors.pithos.object_path
290
    def _run(self):
291
292
293
        r = self.client.container_get(
            limit=False if self['more'] else self['limit'],
            marker=self['marker'],
294
            prefix=self['name_pref'],
295
296
            delimiter=self['delimiter'],
            path=self.path or '',
297
            show_only_shared=self['shared_by_me'],
Stavros Sachtouris's avatar
Stavros Sachtouris committed
298
            public=self['public'],
299
300
301
            if_modified_since=self['if_modified_since'],
            if_unmodified_since=self['if_unmodified_since'],
            until=self['until'],
302
            meta=self['meta'])
303
304
305
306
307
308

        #  REMOVE THIS if version >> 0.12
        if not r.json:
            self.error('  NOTE: Since v0.12, use / for containers e.g.,')
            self.error('    [kamaki] file list /pithos')

309
        files = self._filter_by_name(r.json)
310
311
312
        if self['more']:
            outbu, self._out = self._out, StringIO()
        try:
313
            if self['json_output'] or self['output_format']:
314
315
                self._print(files)
            else:
316
                self.print_objects(files)
317
318
319
320
        finally:
            if self['more']:
                pager(self._out.getvalue())
                self._out = outbu
321

322
    def main(self, path_or_url=''):
323
        super(self.__class__, self)._run(path_or_url)
324
        self._run()
325

Stavros Sachtouris's avatar
Stavros Sachtouris committed
326

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
@command(file_cmds)
class file_modify(_pithos_container):
    """Modify the attributes of a file or directory object"""

    arguments = dict(
        publish=FlagArgument(
            'Make an object public (returns the public URL)', '--publish'),
        unpublish=FlagArgument(
            'Make an object unpublic', '--unpublish'),
        uuid_for_read_permission=RepeatableArgument(
            'Give read access to user/group (can be repeated, accumulative). '
            'Format for users: UUID . Format for groups: UUID:GROUP . '
            'Use * for all users/groups', '--read-permission'),
        uuid_for_write_permission=RepeatableArgument(
            'Give write access to user/group (can be repeated, accumulative). '
            'Format for users: UUID . Format for groups: UUID:GROUP . '
            'Use * for all users/groups', '--write-permission'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
344
345
346
347
348
349
        no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
        metadata_to_set=KeyValueArgument(
            'Add metadata (KEY=VALUE) to an object (can be repeated)',
            '--metadata-add'),
        metadata_key_to_delete=RepeatableArgument(
            'Delete object metadata (can be repeated)', '--metadata-del'),
350
351
    )
    required = [
Stavros Sachtouris's avatar
Stavros Sachtouris committed
352
353
354
        'publish', 'unpublish', 'uuid_for_read_permission', 'metadata_to_set',
        'uuid_for_write_permission', 'no_permissions',
        'metadata_key_to_delete']
355

356
357
358
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
359
    @errors.pithos.object_path
360
    def _run(self):
361
362
363
364
365
366
367
368
369
370
        if self['publish']:
            self.writeln(self.client.publish_object(self.path))
        if self['unpublish']:
            self.client.unpublish_object(self.path)
        if self['uuid_for_read_permission'] or self[
                'uuid_for_write_permission']:
            perms = self.client.get_object_sharing(self.path)
            read, write = perms.get('read', ''), perms.get('write', '')
            read = read.split(',') if read else []
            write = write.split(',') if write else []
371
372
            read += (self['uuid_for_read_permission'] or [])
            write += (self['uuid_for_write_permission'] or [])
373
374
375
376
377
            self.client.set_object_sharing(
                self.path, read_permission=read, write_permission=write)
            self.print_dict(self.client.get_object_sharing(self.path))
        if self['no_permissions']:
            self.client.del_object_sharing(self.path)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
378
        metadata = self['metadata_to_set'] or dict()
379
        for k in (self['metadata_key_to_delete'] or []):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
380
381
382
383
            metadata[k] = ''
        if metadata:
            self.client.set_object_meta(self.path, metadata)
            self.print_dict(self.client.get_object_meta(self.path))
384
385
386
387
388
389

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        if self['publish'] and self['unpublish']:
            raise CLIInvalidArgument(
                'Arguments %s and %s cannot be used together' % (
390
391
                    self.arguments['publish'].lvalue,
                    self.arguments['publish'].lvalue))
392
393
394
395
        if self['no_permissions'] and (
                self['uuid_for_read_permission'] or self[
                    'uuid_for_write_permission']):
            raise CLIInvalidArgument(
396
397
                '%s cannot be used with other permission arguments' % (
                    self.arguments['no_permissions'].lvalue))
398
        self._run()
399

Stavros Sachtouris's avatar
Stavros Sachtouris committed
400

401
402
@command(file_cmds)
class file_create(_pithos_container, _optional_output_cmd):
403
    """Create an empty file"""
404
405
406
407
408

    arguments = dict(
        content_type=ValueArgument(
            'Set content type (default: application/octet-stream)',
            '--content-type',
409
            default='application/octet-stream')
410
411
    )

412
413
414
415
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
416
417
        self._optional_output(
            self.client.create_object(self.path, self['content_type']))
418

419
420
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
421
        self._run()
422
423
424
425
426
427
428
429
430
431
432
433
434


@command(file_cmds)
class file_mkdir(_pithos_container, _optional_output_cmd):
    """Create a directory: /file create --content-type='applcation/directory'
    """

    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
        self._optional_output(self.client.create_directory(self.path))

Stavros Sachtouris's avatar
Stavros Sachtouris committed
435
436
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
437
        self._run()
438
439


440
@command(file_cmds)
441
class file_delete(_pithos_container):
442
    """Delete a file or directory object"""
443

Stavros Sachtouris's avatar
Stavros Sachtouris committed
444
    arguments = dict(
445
446
447
448
449
450
        until_date=DateArgument('remove history until then', '--until'),
        yes=FlagArgument('Do not prompt for permission', '--yes'),
        recursive=FlagArgument(
            'If a directory, empty first', ('-r', '--recursive')),
        delimiter=ValueArgument(
            'delete objects prefixed with <object><delimiter>', '--delimiter')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
451
    )
452

453
454
455
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
456
457
458
459
460
    @errors.pithos.object_path
    def _run(self):
        if self.path:
            if self['yes'] or self.ask_user(
                    'Delete /%s/%s ?' % (self.container, self.path)):
461
                self.client.del_object(
462
463
                    self.path,
                    until=self['until_date'],
464
                    delimiter='/' if self['recursive'] else self['delimiter'])
465
466
467
            else:
                self.error('Aborted')
        else:
468
469
470
471
472
            if self['yes'] or self.ask_user(
                    'Empty container /%s ?' % self.container):
                self.client.container_delete(self.container, delimiter='/')
            else:
                self.error('Aborted')
473

474
475
476
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()
477

Stavros Sachtouris's avatar
Stavros Sachtouris committed
478

479
class _source_destination(_pithos_container, _optional_output_cmd):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
480

481
    sd_arguments = dict(
482
483
484
485
486
487
488
489
        destination_user_uuid=ValueArgument(
            'default: current user uuid', '--to-account'),
        destination_container=ValueArgument(
            'default: pithos', '--to-container'),
        source_prefix=FlagArgument(
            'Transfer all files that are prefixed with SOURCE PATH If the '
            'destination path is specified, replace SOURCE_PATH with '
            'DESTINATION_PATH',
490
            ('-r', '--recursive')),
491
        force=FlagArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
492
493
494
            'Overwrite destination objects, if needed', ('-f', '--force')),
        source_version=ValueArgument(
            'The version of the source object', '--source-version')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
495
    )
496

497
    def __init__(self, arguments={}, auth_base=None, cloud=None):
498
        self.arguments.update(arguments)
499
500
        self.arguments.update(self.sd_arguments)
        super(_source_destination, self).__init__(
501
            self.arguments, auth_base, cloud)
502

503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
    def _report_transfer(self, src, dst, transfer_name):
        if not dst:
            if transfer_name in ('move', ):
                self.error('  delete source directory %s' % src)
            return
        dst_prf = '' if self.account == self.dst_client.account else (
                'pithos://%s' % self.dst_client.account)
        if src:
            src_prf = '' if self.account == self.dst_client.account else (
                    'pithos://%s' % self.account)
            self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
                transfer_name,
                src_prf, self.container, src,
                dst_prf, self.dst_client.container, dst))
        else:
            self.error('  mkdir %s/%s/%s' % (
                dst_prf, self.dst_client.container, dst))
520
521
522

    @errors.generic.all
    @errors.pithos.account
523
524
525
526
527
528
529
    def _src_dst(self, version=None):
        """Preconditions:
        self.account, self.container, self.path
        self.dst_acc, self.dst_con, self.dst_path
        They should all be configured properly
        :returns: [(src_path, dst_path), ...], if src_path is None, create
            destination directory
530
        """
531
        src_objects, dst_objects, pairs = dict(), dict(), []
532
        try:
533
534
535
536
537
538
539
540
            for obj in self.dst_client.list_objects(
                    prefix=self.dst_path or self.path or '/'):
                dst_objects[obj['name']] = obj
        except ClientError as ce:
            if ce.status in (404, ):
                raise CLIError(
                    'Destination container pithos://%s/%s not found' % (
                        self.dst_client.account, self.dst_client.container))
541
            raise ce
542
543
        if self['source_prefix']:
            #  Copy and replace prefixes
544
            for src_obj in self.client.list_objects(prefix=self.path):
545
546
547
548
549
550
551
                src_objects[src_obj['name']] = src_obj
            for src_path, src_obj in src_objects.items():
                dst_path = '%s%s' % (
                    self.dst_path or self.path, src_path[len(self.path):])
                dst_obj = dst_objects.get(dst_path, None)
                if self['force'] or not dst_obj:
                    #  Just do it
552
553
554
555
                    pairs.append((
                        None if self._is_dir(src_obj) else src_path, dst_path))
                    if self._is_dir(src_obj):
                        pairs.append((self.path or dst_path, None))
556
557
558
559
560
561
562
563
564
565
566
567
                elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
                    raise CLIError(
                        'Destination object exists', importance=2, details=[
                            'Failed while transfering:',
                            '    pithos://%s/%s/%s' % (
                                    self.account,
                                    self.container,
                                    src_path),
                            '--> pithos://%s/%s/%s' % (
                                    self.dst_client.account,
                                    self.dst_client.container,
                                    dst_path),
568
569
                            'Use %s to transfer overwrite' % (
                                    self.arguments['force'].lvalue)])
570
        else:
571
572
            #  One object transfer
            try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
573
574
575
576
                src_version_arg = self.arguments.get('source_version', None)
                src_obj = self.client.get_object_info(
                    self.path,
                    version=src_version_arg.value if src_version_arg else None)
577
578
579
580
581
582
            except ClientError as ce:
                if ce.status in (204, ):
                    raise CLIError(
                        'Missing specific path container %s' % self.container,
                        importance=2, details=[
                            'To transfer container contents %s' % (
583
                                self.arguments['source_prefix'].lvalue)])
584
                raise
585
            dst_path = self.dst_path or self.path
Stavros Sachtouris's avatar
Stavros Sachtouris committed
586
            dst_obj = dst_objects.get(dst_path or self.path, None)
587
588
            if self['force'] or not dst_obj:
                pairs.append(
589
590
591
                    (None if self._is_dir(src_obj) else self.path, dst_path))
                if self._is_dir(src_obj):
                    pairs.append((self.path or dst_path, None))
592
593
594
595
596
597
598
            elif self._is_dir(src_obj):
                raise CLIError(
                    'Cannot transfer an application/directory object',
                    importance=2, details=[
                        'The object pithos://%s/%s/%s is a directory' % (
                            self.account,
                            self.container,
599
                            self.path),
600
                        'To recursively copy a directory, use',
601
                        '  %s' % self.arguments['source_prefix'].lvalue,
602
603
604
605
606
607
608
609
610
611
612
                        'To create a file, use',
                        '  /file create  (general purpose)',
                        '  /file mkdir   (a directory object)'])
            else:
                raise CLIError(
                    'Destination object exists',
                    importance=2, details=[
                        'Failed while transfering:',
                        '    pithos://%s/%s/%s' % (
                                self.account,
                                self.container,
613
                                self.path),
614
615
616
617
                        '--> pithos://%s/%s/%s' % (
                                self.dst_client.account,
                                self.dst_client.container,
                                dst_path),
618
619
                        'Use %s to transfer overwrite' % (
                                self.arguments['force'].lvalue)])
620
621
622
623
624
625
626
627
628
629
630
        return pairs

    def _run(self, source_path_or_url, destination_path_or_url=''):
        super(_source_destination, self)._run(source_path_or_url)
        dst_acc, dst_con, dst_path = self._resolve_pithos_url(
            destination_path_or_url)
        self.dst_client = PithosClient(
            base_url=self.client.base_url, token=self.client.token,
            container=self[
                'destination_container'] or dst_con or self.client.container,
            account=self[
631
                'destination_user_uuid'] or dst_acc or self.account)
632
        self.dst_path = dst_path or self.path
633

634

635
636
637
@command(file_cmds)
class file_copy(_source_destination):
    """Copy objects, even between different accounts or containers"""
638
639

    arguments = dict(
640
        public=ValueArgument('publish new object', '--public'),
641
        content_type=ValueArgument(
642
            'change object\'s content type', '--content-type'),
643
        source_version=ValueArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
644
            'The version of the source object', '--object-version')
645
646
    )

647
648
649
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
650
    @errors.pithos.account
651
652
    def _run(self):
        for src, dst in self._src_dst(self['source_version']):
653
654
            self._report_transfer(src, dst, 'copy')
            if src and dst:
655
656
657
658
659
                self.dst_client.copy_object(
                    src_container=self.client.container,
                    src_object=src,
                    dst_container=self.dst_client.container,
                    dst_object=dst,
660
                    source_account=self.client.account,
661
662
663
                    source_version=self['source_version'],
                    public=self['public'],
                    content_type=self['content_type'])
664
665
            elif dst:
                self.dst_client.create_directory(dst)
666
667

    def main(self, source_path_or_url, destination_path_or_url=None):
668
        super(file_copy, self)._run(
669
670
671
672
673
674
675
            source_path_or_url, destination_path_or_url or '')
        self._run()


@command(file_cmds)
class file_move(_source_destination):
    """Move objects, even between different accounts or containers"""
Stavros Sachtouris's avatar
Stavros Sachtouris committed
676
677

    arguments = dict(
678
        public=ValueArgument('publish new object', '--public'),
679
        content_type=ValueArgument(
680
            'change object\'s content type', '--content-type')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
681
    )
682

683
684
685
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
686
687
688
    @errors.pithos.account
    def _run(self):
        for src, dst in self._src_dst():
689
690
            self._report_transfer(src, dst, 'move')
            if src and dst:
691
692
693
694
695
696
697
698
                self.dst_client.move_object(
                    src_container=self.client.container,
                    src_object=src,
                    dst_container=self.dst_client.container,
                    dst_object=dst,
                    source_account=self.account,
                    public=self['public'],
                    content_type=self['content_type'])
699
700
            elif dst:
                self.dst_client.create_directory(dst)
701
            else:
702
                self.client.del_object(src)
703
704
705
706
707

    def main(self, source_path_or_url, destination_path_or_url=None):
        super(file_move, self)._run(
            source_path_or_url, destination_path_or_url or '')
        self._run()
708
709
710
711


@command(file_cmds)
class file_append(_pithos_container, _optional_output_cmd):
712
713
714
715
716
    """Append local file to (existing) remote object
    The remote object should exist.
    If the remote object is a directory, it is transformed into a file.
    In the later case, objects under the directory remain intact.
    """
717

Stavros Sachtouris's avatar
Stavros Sachtouris committed
718
719
    arguments = dict(
        progress_bar=ProgressBarArgument(
720
            'do not show progress bar', ('-N', '--no-progress-bar'),
721
722
            default=False),
        max_threads=IntArgument('default: 1', '--threads'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
723
    )
724

725
726
727
728
729
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self, local_path):
730
731
        if self['max_threads'] > 0:
            self.client.MAX_THREADS = int(self['max_threads'])
732
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
733
        try:
734
735
736
            with open(local_path, 'rb') as f:
                self._optional_output(
                    self.client.append_object(self.path, f, upload_cb))
737
        finally:
738
739
            self._safe_progress_bar_finish(progress_bar)

740
741
    def main(self, local_path, remote_path_or_url):
        super(self.__class__, self)._run(remote_path_or_url)
742
        self._run(local_path)
743

Stavros Sachtouris's avatar
Stavros Sachtouris committed
744

745
746
747
748
749
750
751
752
@command(file_cmds)
class file_truncate(_pithos_container, _optional_output_cmd):
    """Truncate remote file up to size"""

    arguments = dict(
        size_in_bytes=IntArgument('Length of file after truncation', '--size')
    )
    required = ('size_in_bytes', )
753

754
755
756
757
758
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
759
    def _run(self, size):
760
        self._optional_output(self.client.truncate_object(self.path, size))
761

762
763
764
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run(size=self['size_in_bytes'])
765

Stavros Sachtouris's avatar
Stavros Sachtouris committed
766

767
768
769
@command(file_cmds)
class file_overwrite(_pithos_container, _optional_output_cmd):
    """Overwrite part of a remote file"""
770

Stavros Sachtouris's avatar
Stavros Sachtouris committed
771
772
    arguments = dict(
        progress_bar=ProgressBarArgument(
773
            'do not show progress bar', ('-N', '--no-progress-bar'),
774
775
776
            default=False),
        start_position=IntArgument('File position in bytes', '--from'),
        end_position=IntArgument('File position in bytes', '--to')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
777
    )
778
    required = ('start_position', 'end_position')
779

780
781
782
783
784
785
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
    def _run(self, local_path, start, end):
786
        start, end = int(start), int(end)
787
788
        (progress_bar, upload_cb) = self._safe_progress_bar(
            'Overwrite %s bytes' % (end - start))
789
        try:
790
791
792
793
794
795
796
            with open(path.abspath(local_path), 'rb') as f:
                self._optional_output(self.client.overwrite_object(
                    obj=self.path,
                    start=start,
                    end=end,
                    source_file=f,
                    upload_cb=upload_cb))
797
        finally:
798
799
            self._safe_progress_bar_finish(progress_bar)

800
801
    def main(self, local_path, path_or_url):
        super(self.__class__, self)._run(path_or_url)
802
        self.path = self.path or path.basename(local_path)
803
804
805
806
        self._run(
            local_path=local_path,
            start=self['start_position'],
            end=self['end_position'])
807

808

809
810
@command(file_cmds)
class file_upload(_pithos_container, _optional_output_cmd):
811
812
    """Upload a file"""

Stavros Sachtouris's avatar
Stavros Sachtouris committed
813
    arguments = dict(
814
        max_threads=IntArgument('default: 5', '--threads'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
815
        content_encoding=ValueArgument(
816
            'set MIME content type', '--content-encoding'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
817
        content_disposition=ValueArgument(
818
            'specify objects presentation style', '--content-disposition'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
819
        content_type=ValueArgument('specify content type', '--content-type'),
820
        uuid_for_read_permission=RepeatableArgument(
821
822
            'Give read access to a user or group (can be repeated) '
            'Use * for all users',
823
824
            '--read-permission'),
        uuid_for_write_permission=RepeatableArgument(
825
826
            'Give write access to a user or group (can be repeated) '
            'Use * for all users',
827
            '--write-permission'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
828
829
830
        public=FlagArgument('make object publicly accessible', '--public'),
        progress_bar=ProgressBarArgument(
            'do not show progress bar',
831
            ('-N', '--no-progress-bar'),
832
            default=False),
833
834
835
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
        recursive=FlagArgument(
            'Recursively upload directory *contents* + subdirectories',
836
837
838
839
840
841
842
            ('-r', '--recursive')),
        unchunked=FlagArgument(
            'Upload file as one block (not recommended)', '--unchunked'),
        md5_checksum=ValueArgument(
            'Confirm upload with a custom checksum (MD5)', '--etag'),
        use_hashes=FlagArgument(
            'Source file contains hashmap not data', '--source-is-hashmap'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
843
    )
844

845
846
847
848
849
850
851
852
853
854
    def _sharing(self):
        sharing = dict()
        readlist = self['uuid_for_read_permission']
        if readlist:
            sharing['read'] = self['uuid_for_read_permission']
        writelist = self['uuid_for_write_permission']
        if writelist:
            sharing['write'] = self['uuid_for_write_permission']
        return sharing or None

855
856
857
    def _check_container_limit(self, path):
        cl_dict = self.client.get_container_limit()
        container_limit = int(cl_dict['x-container-policy-quota'])
858
859
        r = self.client.container_get()
        used_bytes = sum(int(o['bytes']) for o in r.json)
860
        path_size = get_path_size(path)
861
        if container_limit and path_size > (container_limit - used_bytes):
862
863
            raise CLIError(
                'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
864
865
                    self.client.container,
                    format_size(container_limit),
866
                    format_size(used_bytes),
867
868
                    format_size(path_size),
                    path),
869
                details=[
870
871
872
873
874
875
876
                    'Check accound limit: /file quota',
                    'Check container limit:',
                    '\t/file containerlimit get %s' % self.client.container,
                    'Increase container limit:',
                    '\t/file containerlimit set <new limit> %s' % (
                        self.client.container)])

877
    def _src_dst(self, local_path, remote_path, objlist=None):
878
        lpath = path.abspath(local_path)
879
        short_path = path.basename(path.abspath(local_path))
880
881
        rpath = remote_path or short_path
        if path.isdir(lpath):
882
            if not self['recursive']:
883
                raise CLIError('%s is a directory' % lpath, details=[
884
885
                    'Use %s to upload directories & contents' % (
                        self.arguments['recursive'].lvalue)])
886
887
            robj = self.client.container_get(path=rpath)
            if not self['overwrite']:
888
889
890
                if robj.json:
                    raise CLIError(
                        'Objects/files prefixed as %s already exist' % rpath,
891
                        details=['Existing objects:'] + ['\t/%s/\t%s' % (
892
893
894
895
896
897
898
899
900
901
902
903
904
905
                            o['name'],
                            o['content_type'][12:]) for o in robj.json] + [
                            'Use -f to add, overwrite or resume'])
                else:
                    try:
                        topobj = self.client.get_object_info(rpath)
                        if not self._is_dir(topobj):
                            raise CLIError(
                                'Object /%s/%s exists but not a directory' % (
                                    self.container, rpath),
                                details=['Use -f to overwrite'])
                    except ClientError as ce:
                        if ce.status not in (404, ):
                            raise
906
            self._check_container_limit(lpath)
907
908
909
910
911
912
913
914
            prev = ''
            for top, subdirs, files in walk(lpath):
                if top != prev:
                    prev = top
                    try:
                        rel_path = rpath + top.split(lpath)[1]
                    except IndexError:
                        rel_path = rpath
915
                    self.error('mkdir /%s/%s' % (
916
                        self.client.container, rel_path))
917
918
919
920
                    self.client.create_directory(rel_path)
                for f in files:
                    fpath = path.join(top, f)
                    if path.isfile(fpath):
921
922
923
                        rel_path = rel_path.replace(path.sep, '/')
                        pathfix = f.replace(path.sep, '/')
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
924
                    else:
925
                        self.error('%s is not a regular file' % fpath)
926
        else:
927
            if not path.isfile(lpath):
928
                raise CLIError(('%s is not a regular file' % lpath) if (
929
                    path.exists(lpath)) else '%s does not exist' % lpath)
930
931
932
            try:
                robj = self.client.get_object_info(rpath)
                if remote_path and self._is_dir(robj):
933
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
934
935
                    self.client.get_object_info(rpath)
                if not self['overwrite']:
936
937
938
939
                    raise CLIError(
                        'Object /%s/%s already exists' % (
                            self.container, rpath),
                        details=['use -f to overwrite / resume'])
940
            except ClientError as ce:
941
                if ce.status not in (404, ):
942
                    raise
943
            self._check_container_limit(lpath)
944
            yield open(lpath, 'rb'), rpath
945

946
    def _run(self, local_path, remote_path):
947
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
948
949
        params = dict(
            content_encoding=self['content_encoding'],
Stavros Sachtouris's avatar
Stavros Sachtouris committed
950
951
            content_type=self['content_type'],
            content_disposition=self['content_disposition'],
952
            sharing=self._sharing(),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
953
            public=self['public'])
954
955
956
957
958
        uploaded, container_info_cache = list, dict()
        rpref = 'pithos://%s' if self['account'] else ''
        for f, rpath in self._src_dst(local_path, remote_path):
            self.error('%s --> %s/%s/%s' % (
                f.name, rpref, self.client.container, rpath))
959
960
961
962
            if not (self['content_type'] and self['content_encoding']):
                ctype, cenc = guess_mime_type(f.name)
                params['content_type'] = self['content_type'] or ctype
                params['content_encoding'] = self['content_encoding'] or cenc
963
            if self['unchunked']:
964