pithos.py 76.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# Copyright 2011-2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.command

34
35
from sys import stdout
from time import localtime, strftime
36
from os import path, makedirs, walk
37

Stavros Sachtouris's avatar
Stavros Sachtouris committed
38
from kamaki.cli import command
39
from kamaki.cli.command_tree import CommandTree
40
from kamaki.cli.errors import raiseCLIError, CLISyntaxError
41
from kamaki.cli.utils import (
42
    format_size, to_bytes, print_dict, print_items, pretty_keys,
43
    page_hold, bold, ask_user, get_path_size, print_json)
44
from kamaki.cli.argument import FlagArgument, ValueArgument, IntArgument
45
from kamaki.cli.argument import KeyValueArgument, DateArgument
46
from kamaki.cli.argument import ProgressBarArgument
47
from kamaki.cli.commands import _command_init, errors
48
from kamaki.clients.pithos import PithosClient, ClientError
49
from kamaki.clients.astakos import AstakosClient
50

51
pithos_cmds = CommandTree('file', 'Pithos+/Storage API commands')
52
_commands = [pithos_cmds]
Stavros Sachtouris's avatar
Stavros Sachtouris committed
53

54

Stavros Sachtouris's avatar
Stavros Sachtouris committed
55
56
# Argument functionality

57
class DelimiterArgument(ValueArgument):
58
59
60
    """
    :value type: string
    :value returns: given string or /
61
62
    """

63
64
65
66
    def __init__(self, caller_obj, help='', parsed_name=None, default=None):
        super(DelimiterArgument, self).__init__(help, parsed_name, default)
        self.caller_obj = caller_obj

Stavros Sachtouris's avatar
Stavros Sachtouris committed
67
    @property
68
    def value(self):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
69
        if self.caller_obj['recursive']:
70
71
            return '/'
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
72
73

    @value.setter
74
75
76
    def value(self, newvalue):
        self._value = newvalue

Stavros Sachtouris's avatar
Stavros Sachtouris committed
77

78
79
class SharingArgument(ValueArgument):
    """Set sharing (read and/or write) groups
80
    .
81
    :value type: "read=term1,term2,... write=term1,term2,..."
82
    .
83
    :value returns: {'read':['term1', 'term2', ...],
84
    .   'write':['term1', 'term2', ...]}
85
    """
Stavros Sachtouris's avatar
Stavros Sachtouris committed
86
87

    @property
88
89
    def value(self):
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
90

91
92
93
94
95
96
97
98
99
    @value.setter
    def value(self, newvalue):
        perms = {}
        try:
            permlist = newvalue.split(' ')
        except AttributeError:
            return
        for p in permlist:
            try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
100
                (key, val) = p.split('=')
101
            except ValueError as err:
102
103
104
                raiseCLIError(
                    err,
                    'Error in --sharing',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
105
106
                    details='Incorrect format',
                    importance=1)
107
            if key.lower() not in ('read', 'write'):
108
109
110
                msg = 'Error in --sharing'
                raiseCLIError(err, msg, importance=1, details=[
                    'Invalid permission key %s' % key])
111
            val_list = val.split(',')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
112
113
            if not key in perms:
                perms[key] = []
114
115
116
117
118
            for item in val_list:
                if item not in perms[key]:
                    perms[key].append(item)
        self._value = perms

Stavros Sachtouris's avatar
Stavros Sachtouris committed
119

120
class RangeArgument(ValueArgument):
121
    """
122
123
    :value type: string of the form <start>-<end> where <start> and <end> are
        integers
124
125
126
    :value returns: the input string, after type checking <start> and <end>
    """

Stavros Sachtouris's avatar
Stavros Sachtouris committed
127
    @property
128
129
    def value(self):
        return getattr(self, '_value', self.default)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
130

131
132
133
134
135
    @value.setter
    def value(self, newvalue):
        if newvalue is None:
            self._value = self.default
            return
136
        (start, end) = newvalue.split('-')
137
        (start, end) = (int(start), int(end))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
138
139
140
141
        self._value = '%s-%s' % (start, end)

# Command specs

142

143
class _pithos_init(_command_init):
144
145
    """Initialize a pithos+ kamaki client"""

146
147
148
149
150
151
    @staticmethod
    def _is_dir(remote_dict):
        return 'application/directory' == remote_dict.get(
            'content_type',
            remote_dict.get('content-type', ''))

152
153
    @errors.generic.all
    def _run(self):
154
        self.token = self.config.get('file', 'token')\
Stavros Sachtouris's avatar
Stavros Sachtouris committed
155
            or self.config.get('global', 'token')
156
        self.base_url = self.config.get('file', 'url')\
Stavros Sachtouris's avatar
Stavros Sachtouris committed
157
            or self.config.get('global', 'url')
158
        self._set_account()
159
        self.container = self.config.get('file', 'container')\
Stavros Sachtouris's avatar
Stavros Sachtouris committed
160
            or self.config.get('global', 'container')
161
162
        self.client = PithosClient(
            base_url=self.base_url,
Stavros Sachtouris's avatar
Stavros Sachtouris committed
163
164
            token=self.token,
            account=self.account,
165
            container=self.container)
166
        self._set_log_params()
167
        self._update_max_threads()
168

169
170
171
    def main(self):
        self._run()

172
    def _set_account(self):
173
174
        user = AstakosClient(self.config.get('user', 'url'), self.token)
        self.account = self['account'] or user.term('uuid')
175
176
177

        """Backwards compatibility"""
        self.account = self.account\
178
            or self.config.get('file', 'account')\
179
            or self.config.get('global', 'account')
180

Stavros Sachtouris's avatar
Stavros Sachtouris committed
181

182
class _file_account_command(_pithos_init):
183
184
    """Base class for account level storage commands"""

185
    def __init__(self, arguments={}):
186
        super(_file_account_command, self).__init__(arguments)
187
        self['account'] = ValueArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
188
            'Set user account (not permanent)',
189
            ('-A', '--account'))
190

191
    def _run(self, custom_account=None):
192
        super(_file_account_command, self)._run()
193
194
195
        if custom_account:
            self.client.account = custom_account
        elif self['account']:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
196
            self.client.account = self['account']
197

198
199
200
201
    @errors.generic.all
    def main(self):
        self._run()

Stavros Sachtouris's avatar
Stavros Sachtouris committed
202

203
class _file_container_command(_file_account_command):
204
205
    """Base class for container level storage commands"""

Stavros Sachtouris's avatar
Stavros Sachtouris committed
206
207
    container = None
    path = None
208

209
    def __init__(self, arguments={}):
210
        super(_file_container_command, self).__init__(arguments)
211
        self['container'] = ValueArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
212
            'Set container to work with (temporary)',
213
            ('-C', '--container'))
214

215
    def extract_container_and_path(
216
217
218
            self,
            container_with_path,
            path_is_optional=True):
219
220
221
222
223
224
225
        """Contains all heuristics for deciding what should be used as
        container or path. Options are:
        * user string of the form container:path
        * self.container, self.path variables set by super constructor, or
        explicitly by the caller application
        Error handling is explicit as these error cases happen only here
        """
226
227
228
        try:
            assert isinstance(container_with_path, str)
        except AssertionError as err:
229
230
231
232
            if self['container'] and path_is_optional:
                self.container = self['container']
                self.client.container = self['container']
                return
233
            raiseCLIError(err)
234

235
        user_cont, sep, userpath = container_with_path.partition(':')
236
237

        if sep:
238
            if not user_cont:
239
240
                raiseCLIError(CLISyntaxError(
                    'Container is missing\n',
241
                    details=errors.pithos.container_howto))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
242
            alt_cont = self['container']
243
            if alt_cont and user_cont != alt_cont:
244
                raiseCLIError(CLISyntaxError(
245
246
                    'Conflict: 2 containers (%s, %s)' % (user_cont, alt_cont),
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
247
                )
248
249
            self.container = user_cont
            if not userpath:
250
                raiseCLIError(CLISyntaxError(
251
252
                    'Path is missing for object in container %s' % user_cont,
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
253
                )
254
            self.path = userpath
255
        else:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
256
            alt_cont = self['container'] or self.client.container
257
258
            if alt_cont:
                self.container = alt_cont
259
                self.path = user_cont
260
            elif path_is_optional:
261
                self.container = user_cont
262
263
                self.path = None
            else:
264
                self.container = user_cont
265
                raiseCLIError(CLISyntaxError(
266
                    'Both container and path are required',
267
                    details=errors.pithos.container_howto)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
268
                )
269

270
271
    @errors.generic.all
    def _run(self, container_with_path=None, path_is_optional=True):
272
        super(_file_container_command, self)._run()
273
274
275
276
277
278
279
280
281
        if self['container']:
            self.client.container = self['container']
            if container_with_path:
                self.path = container_with_path
            elif not path_is_optional:
                raise CLISyntaxError(
                    'Both container and path are required',
                    details=errors.pithos.container_howto)
        elif container_with_path:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
282
283
            self.extract_container_and_path(
                container_with_path,
Stavros Sachtouris's avatar
Stavros Sachtouris committed
284
                path_is_optional)
285
286
287
            self.client.container = self.container
        self.container = self.client.container

288
289
290
    def main(self, container_with_path=None, path_is_optional=True):
        self._run(container_with_path, path_is_optional)

291

292
@command(pithos_cmds)
293
class file_list(_file_container_command):
294
    """List containers, object trees or objects in a directory
295
    Use with:
296
    1 no parameters : containers in current account
297
298
    2. one parameter (container) or --container : contents of container
    3. <container>:<prefix> or --container=<container> <prefix>: objects in
299
    .   container starting with prefix
300
301
    """

Stavros Sachtouris's avatar
Stavros Sachtouris committed
302
    arguments = dict(
303
304
305
306
        detail=FlagArgument('detailed output', ('-l', '--list')),
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
        marker=ValueArgument('output greater that marker', '--marker'),
        prefix=ValueArgument('output starting with prefix', '--prefix'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
        path=ValueArgument(
            'show output starting with prefix up to /',
            '--path'),
        meta=ValueArgument(
            'show output with specified meta keys',
            '--meta',
            default=[]),
        if_modified_since=ValueArgument(
            'show output modified since then',
            '--if-modified-since'),
        if_unmodified_since=ValueArgument(
            'show output not modified since then',
            '--if-unmodified-since'),
        until=DateArgument('show metadata until then', '--until'),
        format=ValueArgument(
            'format to parse until data (default: d/m/Y H:M:S )',
            '--format'),
        shared=FlagArgument('show only shared', '--shared'),
326
327
        more=FlagArgument(
            'output results in pages (-n to set items per page, default 10)',
328
329
330
            '--more'),
        exact_match=FlagArgument(
            'Show only objects that match exactly with path',
331
            '--exact-match'),
332
333
        enum=FlagArgument('Enumerate results', '--enumerate'),
        json_output=FlagArgument('show output in json', ('-j', '--json'))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
334
    )
335

336
    def print_objects(self, object_list):
337
338
339
        if self['json_output']:
            print_json(object_list)
            return
340
        limit = int(self['limit']) if self['limit'] > 0 else len(object_list)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
341
        for index, obj in enumerate(object_list):
342
            if self['exact_match'] and self.path and not (
343
344
                    obj['name'] == self.path or 'content_type' in obj):
                continue
345
346
            pretty_obj = obj.copy()
            index += 1
Stavros Sachtouris's avatar
Stavros Sachtouris committed
347
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
348
349
350
351
352
353
            if obj['content_type'] == 'application/directory':
                isDir = True
                size = 'D'
            else:
                isDir = False
                size = format_size(obj['bytes'])
Stavros Sachtouris's avatar
Stavros Sachtouris committed
354
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
355
            oname = bold(obj['name'])
356
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
Stavros Sachtouris's avatar
Stavros Sachtouris committed
357
            if self['detail']:
358
                print('%s%s' % (prfx, oname))
359
360
361
                print_dict(pretty_keys(pretty_obj), exclude=('name'))
                print
            else:
362
                oname = '%s%9s %s' % (prfx, size, oname)
363
364
                oname += '/' if isDir else ''
                print(oname)
365
366
            if self['more']:
                page_hold(index, limit, len(object_list))
367
368

    def print_containers(self, container_list):
369
370
371
        if self['json_output']:
            print_json(container_list)
            return
372
373
        limit = int(self['limit']) if self['limit'] > 0\
            else len(container_list)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
374
375
376
        for index, container in enumerate(container_list):
            if 'bytes' in container:
                size = format_size(container['bytes'])
377
378
            prfx = ('%s. ' % (index + 1)) if self['enum'] else ''
            cname = '%s%s' % (prfx, bold(container['name']))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
379
            if self['detail']:
380
381
                print(cname)
                pretty_c = container.copy()
Stavros Sachtouris's avatar
Stavros Sachtouris committed
382
383
                if 'bytes' in container:
                    pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size)
384
385
386
                print_dict(pretty_keys(pretty_c), exclude=('name'))
                print
            else:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
387
                if 'count' in container and 'bytes' in container:
388
389
390
391
                    print('%s (%s, %s objects)' % (
                        cname,
                        size,
                        container['count']))
392
393
                else:
                    print(cname)
394
395
            if self['more']:
                page_hold(index + 1, limit, len(container_list))
396

397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.object_path
    @errors.pithos.container
    def _run(self):
        if self.container is None:
            r = self.client.account_get(
                limit=False if self['more'] else self['limit'],
                marker=self['marker'],
                if_modified_since=self['if_modified_since'],
                if_unmodified_since=self['if_unmodified_since'],
                until=self['until'],
                show_only_shared=self['shared'])
            self.print_containers(r.json)
        else:
412
            prefix = self.path or self['prefix']
413
414
415
416
417
418
419
420
421
422
423
424
425
            r = self.client.container_get(
                limit=False if self['more'] else self['limit'],
                marker=self['marker'],
                prefix=prefix,
                delimiter=self['delimiter'],
                path=self['path'],
                if_modified_since=self['if_modified_since'],
                if_unmodified_since=self['if_unmodified_since'],
                until=self['until'],
                meta=self['meta'],
                show_only_shared=self['shared'])
            self.print_objects(r.json)

426
    def main(self, container____path__=None):
427
428
        super(self.__class__, self)._run(container____path__)
        self._run()
429

Stavros Sachtouris's avatar
Stavros Sachtouris committed
430

431
@command(pithos_cmds)
432
class file_mkdir(_file_container_command):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
433
    """Create a directory"""
434

435
436
437
438
439
    arguments = dict(
        with_output=FlagArgument('show response headers', ('--with-output')),
        json_output=FlagArgument('show headers in json', ('-j', '--json'))
    )

440
441
442
443
444
445
    __doc__ += '\n. '.join([
        'Kamaki hanldes directories the same way as OOS Storage and Pithos+:',
        'A   directory  is   an  object  with  type  "application/directory"',
        'An object with path  dir/name can exist even if  dir does not exist',
        'or even if dir  is  a non  directory  object.  Users can modify dir',
        'without affecting the dir/name object in any way.'])
446

447
448
449
450
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
451
452
453
454
455
        r = self.client.create_directory(self.path)
        if self['json_output']:
            print_json(r)
        elif self['with_output']:
            print_dict(r)
456

457
    def main(self, container___directory):
458
459
460
461
        super(self.__class__, self)._run(
            container___directory,
            path_is_optional=False)
        self._run()
462

Stavros Sachtouris's avatar
Stavros Sachtouris committed
463

464
@command(pithos_cmds)
465
class file_touch(_file_container_command):
466
467
468
469
470
471
472
473
    """Create an empty object (file)
    If object exists, this command will reset it to 0 length
    """

    arguments = dict(
        content_type=ValueArgument(
            'Set content type (default: application/octet-stream)',
            '--content-type',
474
475
476
            default='application/octet-stream'),
        with_output=FlagArgument('show response headers', ('--with-output')),
        json_output=FlagArgument('show headers in json', ('-j', '--json'))
477
478
    )

479
480
481
482
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
483
484
485
486
487
        r = self.client.create_object(self.path, self['content_type'])
        if self['json_output']:
            print_json(r)
        elif self['with_output']:
            print_dict(r)
488

489
    def main(self, container___path):
490
        super(file_touch, self)._run(
491
492
            container___path,
            path_is_optional=False)
493
        self._run()
494
495
496


@command(pithos_cmds)
497
class file_create(_file_container_command):
498
    """Create a container"""
499

Stavros Sachtouris's avatar
Stavros Sachtouris committed
500
501
502
503
    arguments = dict(
        versioning=ValueArgument(
            'set container versioning (auto/none)',
            '--versioning'),
504
        limit=IntArgument('set default container limit', '--limit'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
505
506
        meta=KeyValueArgument(
            'set container metadata (can be repeated)',
507
508
509
            '--meta'),
        with_output=FlagArgument('show request headers', ('--with-output')),
        json_output=FlagArgument('show headers in json', ('-j', '--json'))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
510
    )
511

512
513
514
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
515
516
517
518
    def _run(self, container):
        r = self.client.create_container(
            container=container,
            sizelimit=self['limit'],
519
520
            versioning=self['versioning'],
            metadata=self['meta'])
521
522
523
524
        if self['json_output']:
            print_json(r)
        elif self['with_output']:
            print_dict(r)
525

526
    def main(self, container=None):
527
        super(self.__class__, self)._run(container)
528
        if container and self.container != container:
529
            raiseCLIError('Invalid container name %s' % container, details=[
530
                'Did you mean "%s" ?' % self.container,
531
                'Use --container for names containing :'])
532
        self._run(container)
533

Stavros Sachtouris's avatar
Stavros Sachtouris committed
534

535
class _source_destination_command(_file_container_command):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
536
537

    arguments = dict(
538
539
        destination_account=ValueArgument('', ('a', '--dst-account')),
        recursive=FlagArgument('', ('-R', '--recursive')),
540
541
542
543
544
        prefix=FlagArgument('', '--with-prefix', default=''),
        suffix=ValueArgument('', '--with-suffix', default=''),
        add_prefix=ValueArgument('', '--add-prefix', default=''),
        add_suffix=ValueArgument('', '--add-suffix', default=''),
        prefix_replace=ValueArgument('', '--prefix-to-replace', default=''),
545
        suffix_replace=ValueArgument('', '--suffix-to-replace', default=''),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
546
    )
547

548
549
550
551
    def __init__(self, arguments={}):
        self.arguments.update(arguments)
        super(_source_destination_command, self).__init__(self.arguments)

552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
    def _run(self, source_container___path, path_is_optional=False):
        super(_source_destination_command, self)._run(
            source_container___path,
            path_is_optional)
        self.dst_client = PithosClient(
            base_url=self.client.base_url,
            token=self.client.token,
            account=self['destination_account'] or self.client.account)

    @errors.generic.all
    @errors.pithos.account
    def _dest_container_path(self, dest_container_path):
        if self['destination_container']:
            self.dst_client.container = self['destination_container']
            return (self['destination_container'], dest_container_path)
        if dest_container_path:
            dst = dest_container_path.split(':')
            if len(dst) > 1:
                try:
                    self.dst_client.container = dst[0]
                    self.dst_client.get_container_info(dst[0])
                except ClientError as err:
                    if err.status in (404, 204):
                        raiseCLIError(
                            'Destination container %s not found' % dst[0])
                    raise
                else:
                    self.dst_client.container = dst[0]
                return (dst[0], dst[1])
            return(None, dst[0])
        raiseCLIError('No destination container:path provided')

584
585
586
    def _get_all(self, prefix):
        return self.client.container_get(prefix=prefix).json

587
    def _get_src_objects(self, src_path, source_version=None):
588
589
590
591
592
593
594
        """Get a list of the source objects to be called

        :param src_path: (str) source path

        :returns: (method, params) a method that returns a list when called
        or (object) if it is a single object
        """
595
596
597
        if src_path and src_path[-1] == '/':
            src_path = src_path[:-1]

598
599
        if self['prefix']:
            return (self._get_all, dict(prefix=src_path))
600
        try:
601
602
            srcobj = self.client.get_object_info(
                src_path, version=source_version)
603
        except ClientError as srcerr:
604
605
            if srcerr.status == 404:
                raiseCLIError(
606
607
608
                    'Source object %s not in source container %s' % (
                        src_path,
                        self.client.container),
609
610
                    details=['Hint: --with-prefix to match multiple objects'])
            elif srcerr.status not in (204,):
611
                raise
612
            return (self.client.list_objects, {})
613

614
615
616
        if self._is_dir(srcobj):
            if not self['recursive']:
                raiseCLIError(
617
618
619
                    'Object %s of cont. %s is a dir' % (
                        src_path,
                        self.client.container),
620
621
622
623
624
                    details=['Use --recursive to access directories'])
            return (self._get_all, dict(prefix=src_path))
        srcobj['name'] = src_path
        return srcobj

625
626
    def src_dst_pairs(self, dst_path, source_version=None):
        src_iter = self._get_src_objects(self.path, source_version)
627
628
629
630
        src_N = isinstance(src_iter, tuple)
        add_prefix = self['add_prefix'].strip('/')

        if dst_path and dst_path.endswith('/'):
631
632
            dst_path = dst_path[:-1]

633
        try:
634
            dstobj = self.dst_client.get_object_info(dst_path)
635
636
637
638
639
640
641
        except ClientError as trgerr:
            if trgerr.status in (404,):
                if src_N:
                    raiseCLIError(
                        'Cannot merge multiple paths to path %s' % dst_path,
                        details=[
                            'Try to use / or a directory as destination',
642
                            'or create the destination dir (/file mkdir)',
643
644
645
646
647
648
649
650
651
652
653
                            'or use a single object as source'])
            elif trgerr.status not in (204,):
                raise
        else:
            if self._is_dir(dstobj):
                add_prefix = '%s/%s' % (dst_path.strip('/'), add_prefix)
            elif src_N:
                raiseCLIError(
                    'Cannot merge multiple paths to path' % dst_path,
                    details=[
                        'Try to use / or a directory as destination',
654
                        'or create the destination dir (/file mkdir)',
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
                        'or use a single object as source'])

        if src_N:
            (method, kwargs) = src_iter
            for obj in method(**kwargs):
                name = obj['name']
                if name.endswith(self['suffix']):
                    yield (name, self._get_new_object(name, add_prefix))
        elif src_iter['name'].endswith(self['suffix']):
            name = src_iter['name']
            yield (name, self._get_new_object(dst_path or name, add_prefix))
        else:
            raiseCLIError('Source path %s conflicts with suffix %s' % (
                src_iter['name'],
                self['suffix']))
670

671
672
673
674
675
676
    def _get_new_object(self, obj, add_prefix):
        if self['prefix_replace'] and obj.startswith(self['prefix_replace']):
            obj = obj[len(self['prefix_replace']):]
        if self['suffix_replace'] and obj.endswith(self['suffix_replace']):
            obj = obj[:-len(self['suffix_replace'])]
        return add_prefix + obj + self['add_suffix']
677

678
679

@command(pithos_cmds)
680
class file_copy(_source_destination_command):
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
    """Copy objects from container to (another) container
    Semantics:
    copy cont:path dir
    .   transfer path as dir/path
    copy cont:path cont2:
    .   trasnfer all <obj> prefixed with path to container cont2
    copy cont:path [cont2:]path2
    .   transfer path to path2
    Use options:
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
    destination is container1:path2
    2. <container>:<path1> <path2> : make a copy in the same container
    3. Can use --container= instead of <container1>
    """

    arguments = dict(
697
698
        destination_account=ValueArgument(
            'Account to copy to',
699
            ('-a', '--dst-account')),
700
701
        destination_container=ValueArgument(
            'use it if destination container name contains a : character',
702
            ('-D', '--dst-container')),
703
704
705
706
707
708
        public=ValueArgument('make object publicly accessible', '--public'),
        content_type=ValueArgument(
            'change object\'s content type',
            '--content-type'),
        recursive=FlagArgument(
            'copy directory and contents',
709
            ('-R', '--recursive')),
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
        prefix=FlagArgument(
            'Match objects prefixed with src path (feels like src_path*)',
            '--with-prefix',
            default=''),
        suffix=ValueArgument(
            'Suffix of source objects (feels like *suffix)',
            '--with-suffix',
            default=''),
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
        prefix_replace=ValueArgument(
            'Prefix of src to replace with dst path + add_prefix, if matched',
            '--prefix-to-replace',
            default=''),
        suffix_replace=ValueArgument(
            'Suffix of src to replace with add_suffix, if matched',
            '--suffix-to-replace',
727
728
729
            default=''),
        source_version=ValueArgument(
            'copy specific version',
730
731
732
            ('-S', '--source-version')),
        with_output=FlagArgument('show request headers', ('--with-output')),
        json_output=FlagArgument('show headers in json', ('-j', '--json'))
733
734
    )

735
736
737
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
738
739
    @errors.pithos.account
    def _run(self, dst_path):
740
        no_source_object = True
741
742
        src_account = self.client.account if (
            self['destination_account']) else None
743
744
        for src_obj, dst_obj in self.src_dst_pairs(
                dst_path, self['source_version']):
745
            no_source_object = False
746
            r = self.dst_client.copy_object(
747
                src_container=self.client.container,
748
                src_object=src_obj,
749
750
751
                dst_container=self.dst_client.container,
                dst_object=dst_obj,
                source_account=src_account,
752
753
754
755
756
757
758
                source_version=self['source_version'],
                public=self['public'],
                content_type=self['content_type'])
        if no_source_object:
            raiseCLIError('No object %s in container %s' % (
                self.path,
                self.container))
759
760
761
762
763
        if self['json_output']:
            print_json(r)
        elif self['with_output']:
            print_dict(r)

764

765
    def main(
766
            self, source_container___path,
767
            destination_container___path=None):
768
        super(file_copy, self)._run(
769
770
771
            source_container___path,
            path_is_optional=False)
        (dst_cont, dst_path) = self._dest_container_path(
772
            destination_container___path)
773
774
        self.dst_client.container = dst_cont or self.container
        self._run(dst_path=dst_path or '')
775

Stavros Sachtouris's avatar
Stavros Sachtouris committed
776

777
@command(pithos_cmds)
778
class file_move(_source_destination_command):
779
    """Move/rename objects from container to (another) container
780
    Semantics:
781
782
    move cont:path dir
    .   rename path as dir/path
783
    move cont:path cont2:
784
785
786
    .   trasnfer all <obj> prefixed with path to container cont2
    move cont:path [cont2:]path2
    .   transfer path to path2
Stavros Sachtouris's avatar
Stavros Sachtouris committed
787
    Use options:
788
    1. <container1>:<path1> [container2:]<path2> : if container2 is not given,
789
    destination is container1:path2
790
    2. <container>:<path1> <path2> : move in the same container
791
    3. Can use --container= instead of <container1>
Stavros Sachtouris's avatar
Stavros Sachtouris committed
792
793
794
    """

    arguments = dict(
795
796
        destination_account=ValueArgument(
            'Account to move to',
797
            ('-a', '--dst-account')),
798
799
        destination_container=ValueArgument(
            'use it if destination container name contains a : character',
800
            ('-D', '--dst-container')),
801
802
803
804
805
806
        public=ValueArgument('make object publicly accessible', '--public'),
        content_type=ValueArgument(
            'change object\'s content type',
            '--content-type'),
        recursive=FlagArgument(
            'copy directory and contents',
807
            ('-R', '--recursive')),
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
        prefix=FlagArgument(
            'Match objects prefixed with src path (feels like src_path*)',
            '--with-prefix',
            default=''),
        suffix=ValueArgument(
            'Suffix of source objects (feels like *suffix)',
            '--with-suffix',
            default=''),
        add_prefix=ValueArgument('Prefix targets', '--add-prefix', default=''),
        add_suffix=ValueArgument('Suffix targets', '--add-suffix', default=''),
        prefix_replace=ValueArgument(
            'Prefix of src to replace with dst path + add_prefix, if matched',
            '--prefix-to-replace',
            default=''),
        suffix_replace=ValueArgument(
            'Suffix of src to replace with add_suffix, if matched',
            '--suffix-to-replace',
825
826
827
            default=''),
        with_output=FlagArgument('show request headers', ('--with-output')),
        json_output=FlagArgument('show headers in json', ('-j', '--json'))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
828
    )
829

830
831
832
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
833
    def _run(self, dst_path):
834
        no_source_object = True
835
836
837
        src_account = self.client.account if (
            self['destination_account']) else None
        for src_obj, dst_obj in self.src_dst_pairs(dst_path):
838
            no_source_object = False
839
            r = self.dst_client.move_object(
840
                src_container=self.container,
841
842
843
                src_object=src_obj,
                dst_container=self.dst_client.container,
                dst_object=dst_obj,
844
                source_account=src_account,
845
846
847
848
849
850
                public=self['public'],
                content_type=self['content_type'])
        if no_source_object:
            raiseCLIError('No object %s in container %s' % (
                self.path,
                self.container))
851
852
853
854
        if self['json_output']:
            print_json(r)
        elif self['with_output']:
            print_dict(r)
855

856
    def main(
857
            self, source_container___path,
858
            destination_container___path=None):
859
860
861
        super(self.__class__, self)._run(
            source_container___path,
            path_is_optional=False)
862
        (dst_cont, dst_path) = self._dest_container_path(
863
            destination_container___path)
864
865
866
867
        (dst_cont, dst_path) = self._dest_container_path(
            destination_container___path)
        self.dst_client.container = dst_cont or self.container
        self._run(dst_path=dst_path or '')
868

Stavros Sachtouris's avatar
Stavros Sachtouris committed
869

870
@command(pithos_cmds)
871
class file_append(_file_container_command):
872
873
874
875
876
    """Append local file to (existing) remote object
    The remote object should exist.
    If the remote object is a directory, it is transformed into a file.
    In the later case, objects under the directory remain intact.
    """
877

Stavros Sachtouris's avatar
Stavros Sachtouris committed
878
879
880
    arguments = dict(
        progress_bar=ProgressBarArgument(
            'do not show progress bar',
881
            ('-N', '--no-progress-bar'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
882
883
            default=False)
    )
884

885
886
887
888
889
890
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self, local_path):
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
891
        try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
892
            f = open(local_path, 'rb')
893
            self.client.append_object(self.path, f, upload_cb)
894
895
896
        except Exception:
            self._safe_progress_bar_finish(progress_bar)
            raise
897
        finally:
898
899
900
901
902
903
904
            self._safe_progress_bar_finish(progress_bar)

    def main(self, local_path, container___path):
        super(self.__class__, self)._run(
            container___path,
            path_is_optional=False)
        self._run(local_path)
905

Stavros Sachtouris's avatar
Stavros Sachtouris committed
906

907
@command(pithos_cmds)
908
class file_truncate(_file_container_command):
909
    """Truncate remote file up to a size (default is 0)"""
910

911
912
913
914
915
916
917
918
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
    def _run(self, size=0):
        self.client.truncate_object(self.path, size)

919
    def main(self, container___path, size=0):
920
921
        super(self.__class__, self)._run(container___path)
        self._run(size=size)
922

Stavros Sachtouris's avatar
Stavros Sachtouris committed
923

924
@command(pithos_cmds)
925
class file_overwrite(_file_container_command):
926
927
928
929
930
931
932
    """Overwrite part (from start to end) of a remote file
    overwrite local-path container 10 20
    .   will overwrite bytes from 10 to 20 of a remote file with the same name
    .   as local-path basename
    overwrite local-path container:path 10 20
    .   will overwrite as above, but the remote file is named path
    """
933

Stavros Sachtouris's avatar
Stavros Sachtouris committed
934
935
936
    arguments = dict(
        progress_bar=ProgressBarArgument(
            'do not show progress bar',
937
            ('-N', '--no-progress-bar'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
938
939
            default=False)
    )
940

941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
    def _open_file(self, local_path, start):
        f = open(path.abspath(local_path), 'rb')
        f.seek(0, 2)
        f_size = f.tell()
        f.seek(start, 0)
        return (f, f_size)

    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
    def _run(self, local_path, start, end):
        (start, end) = (int(start), int(end))
        (f, f_size) = self._open_file(local_path, start)
        (progress_bar, upload_cb) = self._safe_progress_bar(
            'Overwrite %s bytes' % (end - start))
958
        try:
959
960
            self.client.overwrite_object(
                obj=self.path,
Stavros Sachtouris's avatar
Stavros Sachtouris committed
961
962
963
964
                start=start,
                end=end,
                source_file=f,
                upload_cb=upload_cb)