pithos.py 78.3 KB
Newer Older
1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.command

34
35
from sys import stdout
from time import localtime, strftime
36
from os import path, makedirs, walk
37

Stavros Sachtouris's avatar
Stavros Sachtouris committed
38
from kamaki.cli import command
39
from kamaki.cli.command_tree import CommandTree
40
from kamaki.cli.errors import raiseCLIError, CLISyntaxError, CLIBaseUrlError
41
from kamaki.cli.utils import (
42
43
    format_size, to_bytes, print_dict, print_items, page_hold, bold, ask_user,
    get_path_size, print_json, guess_mime_type)
44
from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
45
from kamaki.cli.argument import KeyValueArgument, DateArgument
46
from kamaki.cli.argument import ProgressBarArgument
47
from kamaki.cli.commands import _command_init, errors
48
from kamaki.cli.commands import addLogSettings, DontRaiseKeyError
49
from kamaki.cli.commands import _optional_output_cmd, _optional_json
50
from kamaki.clients.pithos import PithosClient, ClientError
51
from kamaki.clients.astakos import AstakosClient
52

53
pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')
54
_commands = [pithos_cmds]
Stavros Sachtouris's avatar
Stavros Sachtouris committed
55

56

Stavros Sachtouris's avatar
Stavros Sachtouris committed
57
58
# Argument functionality

59
class DelimiterArgument(ValueArgument):
60
61
62
    """
    :value type: string
    :value returns: given string or /
63
64
    """

65
66
67
68
    def __init__(self, caller_obj, help='', parsed_name=None, default=None):
        super(DelimiterArgument, self).__init__(help, parsed_name, default)
        self.caller_obj = caller_obj

Stavros Sachtouris's avatar
Stavros Sachtouris committed
69
    @property
70
    def value(self):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
71
        if self.caller_obj['recursive']:
72
73
            return '/'
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
74
75

    @value.setter
76
77
78
    def value(self, newvalue):
        self._value = newvalue

Stavros Sachtouris's avatar
Stavros Sachtouris committed
79

80
81
class SharingArgument(ValueArgument):
    """Set sharing (read and/or write) groups
82
    .
83
    :value type: "read=term1,term2,... write=term1,term2,..."
84
    .
85
    :value returns: {'read':['term1', 'term2', ...],
86
    .   'write':['term1', 'term2', ...]}
87
    """
Stavros Sachtouris's avatar
Stavros Sachtouris committed
88
89

    @property
90
91
    def value(self):
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
92

93
94
95
96
97
98
99
100
101
    @value.setter
    def value(self, newvalue):
        perms = {}
        try:
            permlist = newvalue.split(' ')
        except AttributeError:
            return
        for p in permlist:
            try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
102
                (key, val) = p.split('=')
103
            except ValueError as err:
104
105
106
                raiseCLIError(
                    err,
                    'Error in --sharing',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
107
108
                    details='Incorrect format',
                    importance=1)
109
            if key.lower() not in ('read', 'write'):
110
111
112
                msg = 'Error in --sharing'
                raiseCLIError(err, msg, importance=1, details=[
                    'Invalid permission key %s' % key])
113
            val_list = val.split(',')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
114
115
            if not key in perms:
                perms[key] = []
116
117
118
119
120
            for item in val_list:
                if item not in perms[key]:
                    perms[key].append(item)
        self._value = perms

Stavros Sachtouris's avatar
Stavros Sachtouris committed
121

122
class RangeArgument(ValueArgument):
123
    """
124
125
    :value type: string of the form <start>-<end> where <start> and <end> are
        integers
126
127
128
    :value returns: the input string, after type checking <start> and <end>
    """

Stavros Sachtouris's avatar
Stavros Sachtouris committed
129
    @property
130
131
    def value(self):
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
132

133
    @value.setter
134
135
    def value(self, newvalues):
        if not newvalues:
136
137
            self._value = self.default
            return
138
139
140
141
142
143
144
145
146
147
148
        self._value = ''
        for newvalue in newvalues.split(','):
            self._value = ('%s,' % self._value) if self._value else ''
            start, sep, end = newvalue.partition('-')
            if sep:
                if start:
                    start, end = (int(start), int(end))
                    assert start <= end, 'Invalid range value %s' % newvalue
                    self._value += '%s-%s' % (int(start), int(end))
                else:
                    self._value += '-%s' % int(end)
149
            else:
150
                self._value += '%s' % int(start)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
151

152

Stavros Sachtouris's avatar
Stavros Sachtouris committed
153
154
# Command specs

155

156
class _pithos_init(_command_init):
157
158
    """Initialize a pithos+ kamaki client"""

159
160
161
    @staticmethod
    def _is_dir(remote_dict):
        return 'application/directory' == remote_dict.get(
162
            'content_type', remote_dict.get('content-type', ''))
163

164
165
    @DontRaiseKeyError
    def _custom_container(self):
166
        return self.config.get_cloud(self.cloud, 'pithos_container')
167
168
169

    @DontRaiseKeyError
    def _custom_uuid(self):
170
        return self.config.get_cloud(self.cloud, 'pithos_uuid')
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185

    def _set_account(self):
        self.account = self._custom_uuid()
        if self.account:
            return
        if getattr(self, 'auth_base', False):
            self.account = self.auth_base.user_term('id', self.token)
        else:
            astakos_url = self._custom_url('astakos')
            astakos_token = self._custom_token('astakos') or self.token
            if not astakos_url:
                raise CLIBaseUrlError(service='astakos')
            astakos = AstakosClient(astakos_url, astakos_token)
            self.account = astakos.user_term('id')

186
    @errors.generic.all
187
    @addLogSettings
188
    def _run(self):
189
190
191
192
193
194
195
        self.base_url = None
        if getattr(self, 'cloud', None):
            self.base_url = self._custom_url('pithos')
        else:
            self.cloud = 'default'
        self.token = self._custom_token('pithos')
        self.container = self._custom_container()
196
197

        if getattr(self, 'auth_base', False):
198
199
200
201
202
203
204
            self.token = self.token or self.auth_base.token
            if not self.base_url:
                pithos_endpoints = self.auth_base.get_service_endpoints(
                    self._custom_type('pithos') or 'object-store',
                    self._custom_version('pithos') or '')
                self.base_url = pithos_endpoints['publicURL']
        elif not self.base_url:
205
206
            raise CLIBaseUrlError(service='pithos')

207
        self._set_account()
208
209
        self.client = PithosClient(
            base_url=self.base_url,
Stavros Sachtouris's avatar
Stavros Sachtouris committed
210
211
            token=self.token,
            account=self.account,
212
213
            container=self.container)

214
215
216
    def main(self):
        self._run()

Stavros Sachtouris's avatar
Stavros Sachtouris committed
217

218
class _file_account_command(_pithos_init):
219
220
    """Base class for account level storage commands"""

221
222
223
    def __init__(self, arguments={}, auth_base=None, cloud=None):
        super(_file_account_command, self).__init__(
            arguments, auth_base, cloud)
224
        self['account'] = ValueArgument(
225
            'Set user account (not permanent)', ('-A', '--account'))
226

227
    def _run(self, custom_account=None):
228
        super(_file_account_command, self)._run()
229
230
231
        if custom_account:
            self.client.account = custom_account
        elif self['account']:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
232
            self.client.account = self['account']
233

234
235
236
237
    @errors.generic.all
    def main(self):
        self._run()

Stavros Sachtouris's avatar
Stavros Sachtouris committed
238

239
class _file_container_command(_file_account_command):
240
241
    """Base class for container level storage commands"""

Stavros Sachtouris's avatar
Stavros Sachtouris committed
242
243
    container = None
    path = None
244

245
246
247
    def __init__(self, arguments={}, auth_base=None, cloud=None):
        super(_file_container_command, self).__init__(
            arguments, auth_base, cloud)
248
        self['container'] = ValueArgument(
249
            'Set container to work with (temporary)', ('-C', '--container'))
250

251
    def extract_container_and_path(
252
253
254
            self,
            container_with_path,
            path_is_optional=True):
255
256
257
258
259
260
261
        """Contains all heuristics for deciding what should be used as
        container or path. Options are:
        * user string of the form container:path
        * self.container, self.path variables set by super constructor, or
        explicitly by the caller application
        Error handling is explicit as these error cases happen only here
        """
262
263
264
        try:
            assert isinstance(container_with_path, str)
        except AssertionError as err:
265
266
267
268
            if self['container'] and path_is_optional:
                self.container = self['container']
                self.client.container = self['container']
                return
269
            raiseCLIError(err)
270

271
        user_cont, sep, userpath = container_with_path.partition(':')
272
273

        if sep:
274
            if not user_cont:
275
276
                raiseCLIError(CLISyntaxError(
                    'Container is missing\n',
277
                    details=errors.pithos.container_howto))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
278
            alt_cont = self['container']
279
            if alt_cont and user_cont != alt_cont:
280
                raiseCLIError(CLISyntaxError(
281
282
                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
283
                )
284
285
            self.container = user_cont
            if not userpath:
286
                raiseCLIError(CLISyntaxError(
287
288
                    'Path is missing for object in container %s' % user_cont,
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
289
                )
290
            self.path = userpath
291
        else:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
292
            alt_cont = self['container'] or self.client.container
293
294
            if alt_cont:
                self.container = alt_cont
295
                self.path = user_cont
296
            elif path_is_optional:
297
                self.container = user_cont
298
299
                self.path = None
            else:
300
                self.container = user_cont
301
                raiseCLIError(CLISyntaxError(
302
                    'Both container and path are required',
303
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
304
                )
305

306
307
    @errors.generic.all
    def _run(self, container_with_path=None, path_is_optional=True):
308
        super(_file_container_command, self)._run()
309
310
311
312
313
314
315
316
317
        if self['container']:
            self.client.container = self['container']
            if container_with_path:
                self.path = container_with_path
            elif not path_is_optional:
                raise CLISyntaxError(
                    'Both container and path are required',
                    details=errors.pithos.container_howto)
        elif container_with_path:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
318
319
            self.extract_container_and_path(
                container_with_path,
Stavros Sachtouris's avatar
Stavros Sachtouris committed
320
                path_is_optional)
321
322
323
            self.client.container = self.container
        self.container = self.client.container

324
325
326
    def main(self, container_with_path=None, path_is_optional=True):
        self._run(container_with_path, path_is_optional)

327

328
@command(pithos_cmds)
329
class file_list(_file_container_command, _optional_json):
330
    """List containers, object trees or objects in a directory
331
    Use with:
332
    1 no parameters : containers in current account
333
334
    2. one parameter (container) or --container : contents of container
    3. <container>:<prefix> or --container=<container> <prefix>: objects in
335
    .   container starting with prefix
336
337
    """

Stavros Sachtouris's avatar
Stavros Sachtouris committed
338
    arguments = dict(
339
340
341
342
        detail=FlagArgument('detailed output', ('-l', '--list')),
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
        marker=ValueArgument('output greater that marker', '--marker'),
        prefix=ValueArgument('output starting with prefix', '--prefix'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
343
344
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
        path=ValueArgument(
345
            'show output starting with prefix up to /', '--path'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
346
        meta=ValueArgument(
347
            'show output with specified meta keys', '--meta',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
348
349
            default=[]),
        if_modified_since=ValueArgument(
350
            'show output modified since then', '--if-modified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
351
        if_unmodified_since=ValueArgument(
352
            'show output not modified since then', '--if-unmodified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
353
354
        until=DateArgument('show metadata until then', '--until'),
        format=ValueArgument(
355
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
356
        shared=FlagArgument('show only shared', '--shared'),
357
358
        more=FlagArgument(
            'output results in pages (-n to set items per page, default 10)',
359
360
361
            '--more'),
        exact_match=FlagArgument(
            'Show only objects that match exactly with path',
362
            '--exact-match'),
363
        enum=FlagArgument('Enumerate results', '--enumerate')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
364
    )
365

366
    def print_objects(self, object_list):
367
368
369
        if self['json_output']:
            print_json(object_list)
            return
370
        limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
371
        for index, obj in enumerate(object_list):
372
            if self['exact_match'] and self.path and not (
373
374
                    obj['name'] == self.path or 'content_type' in obj):
                continue
375
376
            pretty_obj = obj.copy()
            index += 1
Stavros Sachtouris's avatar
Stavros Sachtouris committed
377
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
378
379
            if 'subdir' in obj:
                continue
380
381
382
383
384
385
            if obj['content_type'] == 'application/directory':
                isDir = True
                size = 'D'
            else:
                isDir = False
                size = format_size(obj['bytes'])
Stavros Sachtouris's avatar
Stavros Sachtouris committed
386
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
387
            oname = bold(obj['name'])
388
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
Stavros Sachtouris's avatar
Stavros Sachtouris committed
389
            if self['detail']:
390
                print('%s%s' % (prfx, oname))
391
                print_dict(pretty_obj, exclude=('name'))
392
393
                print
            else:
394
                oname = '%s%9s %s' % (prfx, size, oname)
395
396
                oname += '/' if isDir else ''
                print(oname)
397
398
            if self['more']:
                page_hold(index, limit, len(object_list))
399
400

    def print_containers(self, container_list):
401
402
403
        if self['json_output']:
            print_json(container_list)
            return
404
405
        limit = int(self['limit']) if self['limit'] > 0\
            else len(container_list)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
406
407
408
        for index, container in enumerate(container_list):
            if 'bytes' in container:
                size = format_size(container['bytes'])
409
410
            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
            cname = '%s%s' % (prfx, bold(container['name']))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
411
            if self['detail']:
412
413
                print(cname)
                pretty_c = container.copy()
Stavros Sachtouris's avatar
Stavros Sachtouris committed
414
415
                if 'bytes' in container:
                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
416
                print_dict(pretty_c, exclude=('name'))
417
418
                print
            else:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
419
                if 'count' in container and 'bytes' in container:
420
421
422
423
                    print('%s (%s, %s objects)' % (
                        cname,
                        size,
                        container['count']))
424
425
                else:
                    print(cname)
426
427
            if self['more']:
                page_hold(index + 1, limit, len(container_list))
428

429
430
431
432
433
434
435
436
437
438
439
440
441
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.object_path
    @errors.pithos.container
    def _run(self):
        if self.container is None:
            r = self.client.account_get(
                limit=False if self['more'] else self['limit'],
                marker=self['marker'],
                if_modified_since=self['if_modified_since'],
                if_unmodified_since=self['if_unmodified_since'],
                until=self['until'],
                show_only_shared=self['shared'])
442
            self._print(r.json, self.print_containers)
443
        else:
444
            prefix = self.path or self['prefix']
445
446
447
448
449
450
451
452
453
454
455
            r = self.client.container_get(
                limit=False if self['more'] else self['limit'],
                marker=self['marker'],
                prefix=prefix,
                delimiter=self['delimiter'],
                path=self['path'],
                if_modified_since=self['if_modified_since'],
                if_unmodified_since=self['if_unmodified_since'],
                until=self['until'],
                meta=self['meta'],
                show_only_shared=self['shared'])
456
            self._print(r.json, self.print_objects)
457

458
    def main(self, container____path__=None):
459
460
        super(self.__class__, self)._run(container____path__)
        self._run()
461

Stavros Sachtouris's avatar
Stavros Sachtouris committed
462

463
@command(pithos_cmds)
464
class file_mkdir(_file_container_command, _optional_output_cmd):
465
466
467
468
469
470
471
    """Create a directory
    Kamaki hanldes directories the same way as OOS Storage and Pithos+:
    A directory  is   an  object  with  type  "application/directory"
    An object with path  dir/name can exist even if  dir does not exist
    or even if dir  is  a non  directory  object.  Users can modify dir '
    without affecting the dir/name object in any way.
    """
472

473
474
475
476
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
477
        self._optional_output(self.client.create_directory(self.path))
478

479
    def main(self, container___directory):
480
481
482
483
        super(self.__class__, self)._run(
            container___directory,
            path_is_optional=False)
        self._run()
484

Stavros Sachtouris's avatar
Stavros Sachtouris committed
485

486
@command(pithos_cmds)
487
class file_touch(_file_container_command, _optional_output_cmd):
488
489
490
491
492
493
494
495
    """Create an empty object (file)
    If object exists, this command will reset it to 0 length
    """

    arguments = dict(
        content_type=ValueArgument(
            'Set content type (default: application/octet-stream)',
            '--content-type',
496
            default='application/octet-stream')
497
498
    )

499
500
501
502
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
503
504
        self._optional_output(
            self.client.create_object(self.path, self['content_type']))
505

506
    def main(self, container___path):
507
        super(file_touch, self)._run(
508
509
            container___path,
            path_is_optional=False)
510
        self._run()
511
512
513


@command(pithos_cmds)
514
class file_create(_file_container_command, _optional_output_cmd):
515
    """Create a container"""
516

Stavros Sachtouris's avatar
Stavros Sachtouris committed
517
518
    arguments = dict(
        versioning=ValueArgument(
519
            'set container versioning (auto/none)', '--versioning'),
520
        limit=IntArgument('set default container limit', '--limit'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
521
        meta=KeyValueArgument(
522
            'set container metadata (can be repeated)', '--meta')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
523
    )
524

525
526
527
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
528
    def _run(self, container):
529
530
531
532
533
        self._optional_output(self.client.create_container(
            container=container,
            sizelimit=self['limit'],
            versioning=self['versioning'],
            metadata=self['meta']))
534

535
    def main(self, container=None):
536
        super(self.__class__, self)._run(container)
537
        if container and self.container != container:
538
            raiseCLIError('Invalid container name %s' % container, details=[
539
                'Did you mean "%s" ?' % self.container,
540
                'Use --container for names containing :'])
541
        self._run(container)
542

Stavros Sachtouris's avatar
Stavros Sachtouris committed
543

544
class _source_destination_command(_file_container_command):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
545
546

    arguments = dict(
547
        destination_account=ValueArgument('', ('-a', '--dst-account')),
548
        recursive=FlagArgument('', ('-R', '--recursive')),
549
550
551
552
553
        prefix=FlagArgument('', '--with-prefix', default=''),
        suffix=ValueArgument('', '--with-suffix', default=''),
        add_prefix=ValueArgument('', '--add-prefix', default=''),
        add_suffix=ValueArgument('', '--add-suffix', default=''),
        prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
554
        suffix_replace=ValueArgument('', '--suffix-to-replace', default=''),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
555
    )
556

557
    def __init__(self, arguments={}, auth_base=None, cloud=None):
558
        self.arguments.update(arguments)
559
        super(_source_destination_command, self).__init__(
560
            self.arguments, auth_base, cloud)
561

562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
    def _run(self, source_container___path, path_is_optional=False):
        super(_source_destination_command, self)._run(
            source_container___path,
            path_is_optional)
        self.dst_client = PithosClient(
            base_url=self.client.base_url,
            token=self.client.token,
            account=self['destination_account'] or self.client.account)

    @errors.generic.all
    @errors.pithos.account
    def _dest_container_path(self, dest_container_path):
        if self['destination_container']:
            self.dst_client.container = self['destination_container']
            return (self['destination_container'], dest_container_path)
        if dest_container_path:
            dst = dest_container_path.split(':')
            if len(dst) > 1:
                try:
                    self.dst_client.container = dst[0]
                    self.dst_client.get_container_info(dst[0])
                except ClientError as err:
                    if err.status in (404, 204):
                        raiseCLIError(
                            'Destination container %s not found' % dst[0])
                    raise
                else:
                    self.dst_client.container = dst[0]
                return (dst[0], dst[1])
            return(None, dst[0])
        raiseCLIError('No destination container:path provided')

594
595
596
    def _get_all(self, prefix):
        return self.client.container_get(prefix=prefix).json

597
    def _get_src_objects(self, src_path, source_version=None):
598
599
600
601
602
603
604
        """Get a list of the source objects to be called

        :param src_path: (str) source path

        :returns: (method, params) a method that returns a list when called
        or (object) if it is a single object
        """
605
606
607
        if src_path and src_path[-1] == '/':
            src_path = src_path[:-1]

608
609
        if self['prefix']:
            return (self._get_all, dict(prefix=src_path))
610
        try:
611
612
            srcobj = self.client.get_object_info(
                src_path, version=source_version)
613
        except ClientError as srcerr:
614
615
            if srcerr.status == 404:
                raiseCLIError(
616
                    'Source object %s not in source container %s' % (
617
                        src_path, self.client.container),
618
619
                    details=['Hint: --with-prefix to match multiple objects'])
            elif srcerr.status not in (204,):
620
                raise
621
            return (self.client.list_objects, {})
622

623
624
625
        if self._is_dir(srcobj):
            if not self['recursive']:
                raiseCLIError(
626
                    'Object %s of cont. %s is a dir' % (
627
                        src_path, self.client.container),
628
629
630
631
632
                    details=['Use --recursive to access directories'])
            return (self._get_all, dict(prefix=src_path))
        srcobj['name'] = src_path
        return srcobj

633
634
    def src_dst_pairs(self, dst_path, source_version=None):
        src_iter = self._get_src_objects(self.path, source_version)
635
636
637
638
        src_N = isinstance(src_iter, tuple)
        add_prefix = self['add_prefix'].strip('/')

        if dst_path and dst_path.endswith('/'):
639
640
            dst_path = dst_path[:-1]

641
        try:
642
            dstobj = self.dst_client.get_object_info(dst_path)
643
644
645
646
647
648
649
        except ClientError as trgerr:
            if trgerr.status in (404,):
                if src_N:
                    raiseCLIError(
                        'Cannot merge multiple paths to path %s' % dst_path,
                        details=[
                            'Try to use / or a directory as destination',
650
                            'or create the destination dir (/file mkdir)',
651
652
653
654
655
656
657
658
659
660
661
                            'or use a single object as source'])
            elif trgerr.status not in (204,):
                raise
        else:
            if self._is_dir(dstobj):
                add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
            elif src_N:
                raiseCLIError(
                    'Cannot merge multiple paths to path' % dst_path,
                    details=[
                        'Try to use / or a directory as destination',
662
                        'or create the destination dir (/file mkdir)',
663
664
665
666
667
668
669
670
671
672
673
674
675
                        'or use a single object as source'])

        if src_N:
            (method, kwargs) = src_iter
            for obj in method(**kwargs):
                name = obj['name']
                if name.endswith(self['suffix']):
                    yield (name, self._get_new_object(name, add_prefix))
        elif src_iter['name'].endswith(self['suffix']):
            name = src_iter['name']
            yield (name, self._get_new_object(dst_path or name, add_prefix))
        else:
            raiseCLIError('Source path %s conflicts with suffix %s' % (
676
                src_iter['name'], self['suffix']))
677

678
679
680
681
682
683
    def _get_new_object(self, obj, add_prefix):
        if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
            obj = obj[len(self['prefix_replace']):]
        if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
            obj = obj[:-len(self['suffix_replace'])]
        return add_prefix + obj + self['add_suffix']
684

685
686

@command(pithos_cmds)
687
class file_copy(_source_destination_command, _optional_output_cmd):
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
    """Copy objects from container to (another) container
    Semantics:
    copy cont:path dir
    .   transfer path as dir/path
    copy cont:path cont2:
    .   trasnfer all <obj> prefixed with path to container cont2
    copy cont:path [cont2:]path2
    .   transfer path to path2
    Use options:
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
    destination is container1:path2
    2. <container>:<path1> <path2> : make a copy in the same container
    3. Can use --container= instead of <container1>
    """

    arguments = dict(
704
        destination_account=ValueArgument(
705
            'Account to copy to', ('-a', '--dst-account')),
706
707
        destination_container=ValueArgument(
            'use it if destination container name contains a : character',
708
            ('-D', '--dst-container')),
709
710
        public=ValueArgument('make object publicly accessible', '--public'),
        content_type=ValueArgument(
711
            'change object\'s content type', '--content-type'),
712
        recursive=FlagArgument(
713
            'copy directory and contents', ('-R', '--recursive')),
714
715
716
717
718
        prefix=FlagArgument(
            'Match objects prefixed with src path (feels like src_path*)',
            '--with-prefix',
            default=''),
        suffix=ValueArgument(
719
            'Suffix of source objects (feels like *suffix)', '--with-suffix',
720
721
722
723
724
725
726
727
728
729
            default=''),
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
        prefix_replace=ValueArgument(
            'Prefix of src to replace with dst path + add_prefix, if matched',
            '--prefix-to-replace',
            default=''),
        suffix_replace=ValueArgument(
            'Suffix of src to replace with add_suffix, if matched',
            '--suffix-to-replace',
730
731
            default=''),
        source_version=ValueArgument(
732
            'copy specific version', ('-S', '--source-version'))
733
734
    )

735
736
737
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
738
739
    @errors.pithos.account
    def _run(self, dst_path):
740
        no_source_object = True
741
742
        src_account = self.client.account if (
            self['destination_account']) else None
743
744
        for src_obj, dst_obj in self.src_dst_pairs(
                dst_path, self['source_version']):
745
            no_source_object = False
746
            r = self.dst_client.copy_object(
747
                src_container=self.client.container,
748
                src_object=src_obj,
749
750
751
                dst_container=self.dst_client.container,
                dst_object=dst_obj,
                source_account=src_account,
752
753
754
755
756
                source_version=self['source_version'],
                public=self['public'],
                content_type=self['content_type'])
        if no_source_object:
            raiseCLIError('No object %s in container %s' % (
757
                self.path, self.container))
758
        self._optional_output(r)
759

760
    def main(
761
            self, source_container___path,
762
            destination_container___path=None):
763
        super(file_copy, self)._run(
764
765
766
            source_container___path,
            path_is_optional=False)
        (dst_cont, dst_path) = self._dest_container_path(
767
            destination_container___path)
768
769
        self.dst_client.container = dst_cont or self.container
        self._run(dst_path=dst_path or '')
770

Stavros Sachtouris's avatar
Stavros Sachtouris committed
771

772
@command(pithos_cmds)
773
class file_move(_source_destination_command, _optional_output_cmd):
774
    """Move/rename objects from container to (another) container
775
    Semantics:
776
777
    move cont:path dir
    .   rename path as dir/path
778
    move cont:path cont2:
779
780
781
    .   trasnfer all <obj> prefixed with path to container cont2
    move cont:path [cont2:]path2
    .   transfer path to path2
Stavros Sachtouris's avatar
Stavros Sachtouris committed
782
    Use options:
783
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
784
    destination is container1:path2
785
    2. <container>:<path1> <path2> : move in the same container
786
    3. Can use --container= instead of <container1>
Stavros Sachtouris's avatar
Stavros Sachtouris committed
787
788
789
    """

    arguments = dict(
790
        destination_account=ValueArgument(
791
            'Account to move to', ('-a', '--dst-account')),
792
793
        destination_container=ValueArgument(
            'use it if destination container name contains a : character',
794
            ('-D', '--dst-container')),
795
796
        public=ValueArgument('make object publicly accessible', '--public'),
        content_type=ValueArgument(
797
            'change object\'s content type', '--content-type'),
798
        recursive=FlagArgument(
799
            'copy directory and contents', ('-R', '--recursive')),
800
801
802
803
804
        prefix=FlagArgument(
            'Match objects prefixed with src path (feels like src_path*)',
            '--with-prefix',
            default=''),
        suffix=ValueArgument(
805
            'Suffix of source objects (feels like *suffix)', '--with-suffix',
806
807
808
809
810
811
812
813
814
815
            default=''),
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
        prefix_replace=ValueArgument(
            'Prefix of src to replace with dst path + add_prefix, if matched',
            '--prefix-to-replace',
            default=''),
        suffix_replace=ValueArgument(
            'Suffix of src to replace with add_suffix, if matched',
            '--suffix-to-replace',
816
            default='')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
817
    )
818

819
820
821
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
822
    def _run(self, dst_path):
823
        no_source_object = True
824
825
826
        src_account = self.client.account if (
            self['destination_account']) else None
        for src_obj, dst_obj in self.src_dst_pairs(dst_path):
827
            no_source_object = False
828
            r = self.dst_client.move_object(
829
                src_container=self.container,
830
831
832
                src_object=src_obj,
                dst_container=self.dst_client.container,
                dst_object=dst_obj,
833
                source_account=src_account,
834
835
836
837
838
839
                public=self['public'],
                content_type=self['content_type'])
        if no_source_object:
            raiseCLIError('No object %s in container %s' % (
                self.path,
                self.container))
840
        self._optional_output(r)
841

842
    def main(
843
            self, source_container___path,
844
            destination_container___path=None):
845
846
847
        super(self.__class__, self)._run(
            source_container___path,
            path_is_optional=False)
848
        (dst_cont, dst_path) = self._dest_container_path(
849
            destination_container___path)
850
851
852
853
        (dst_cont, dst_path) = self._dest_container_path(
            destination_container___path)
        self.dst_client.container = dst_cont or self.container
        self._run(dst_path=dst_path or '')
854

Stavros Sachtouris's avatar
Stavros Sachtouris committed
855

856
@command(pithos_cmds)
857
class file_append(_file_container_command, _optional_output_cmd):
858
859
860
861
862
    """Append local file to (existing) remote object
    The remote object should exist.
    If the remote object is a directory, it is transformed into a file.
    In the later case, objects under the directory remain intact.
    """
863

Stavros Sachtouris's avatar
Stavros Sachtouris committed
864
865
866
    arguments = dict(
        progress_bar=ProgressBarArgument(
            'do not show progress bar',
867
            ('-N', '--no-progress-bar'),
868
            default=False)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
869
    )
870

871
872
873
874
875
876
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self, local_path):
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
877
        try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
878
            f = open(local_path, 'rb')
879
880
            self._optional_output(
                self.client.append_object(self.path, f, upload_cb))
881
882
883
        except Exception:
            self._safe_progress_bar_finish(progress_bar)
            raise
884
        finally:
885
886
887
888
            self._safe_progress_bar_finish(progress_bar)

    def main(self, local_path, container___path):
        super(self.__class__, self)._run(
889
            container___path, path_is_optional=False)
890
        self._run(local_path)
891

Stavros Sachtouris's avatar
Stavros Sachtouris committed
892

893