pithos.py 69.7 KB
Newer Older
1
# Copyright 2011-2014 GRNET S.A. All rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.command

34
from time import localtime, strftime
35
from io import StringIO
36
from pydoc import pager
37
from os import path, walk, makedirs
38

39
from kamaki.clients.pithos import PithosClient, ClientError
40

Stavros Sachtouris's avatar
Stavros Sachtouris committed
41
from kamaki.cli import command
42
from kamaki.cli.command_tree import CommandTree
43
from kamaki.cli.commands import (
44
45
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
    _name_filter, _optional_output_cmd)
46
from kamaki.cli.errors import (
47
48
    CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
    CLISyntaxError)
49
from kamaki.cli.argument import (
Stavros Sachtouris's avatar
Stavros Sachtouris committed
50
    FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
51
52
    ProgressBarArgument, RepeatableArgument, DataSizeArgument,
    UserAccountArgument)
53
from kamaki.cli.utils import (
54
    format_size, bold, get_path_size, guess_mime_type)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
55

56
57
58
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
container_cmds = CommandTree(
    'container', 'Pithos+/Storage container level API commands')
59
60
61
sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
_commands = [file_cmds, container_cmds, sharer_cmds, group_cmds]
Stavros Sachtouris's avatar
Stavros Sachtouris committed
62

63

64
class _pithos_init(_command_init):
65
66
67
68
    """Initilize a pithos+ client
    There is always a default account (current user uuid)
    There is always a default container (pithos)
    """
69

70
71
    @DontRaiseKeyError
    def _custom_container(self):
72
        return self.config.get_cloud(self.cloud, 'pithos_container')
73
74
75

    @DontRaiseKeyError
    def _custom_uuid(self):
76
        return self.config.get_cloud(self.cloud, 'pithos_uuid')
77
78
79
80
81

    def _set_account(self):
        self.account = self._custom_uuid()
        if self.account:
            return
82
83
84
        astakos = getattr(self, 'auth_base', None)
        if astakos:
            self.account = astakos.user_term('id', self.token)
85
        else:
86
            raise CLIBaseUrlError(service='astakos')
87

88
    @errors.generic.all
89
    @addLogSettings
90
    def _run(self):
91
92
        self.client = self.get_client(PithosClient, 'pithos')
        self.base_url, self.token = self.client.base_url, self.client.token
93
        self._set_account()
94
95
96
        self.client.account = self.account
        self.container = self._custom_container() or 'pithos'
        self.client.container = self.container
97

98
99
100
    def main(self):
        self._run()

Stavros Sachtouris's avatar
Stavros Sachtouris committed
101

102
103
class _pithos_account(_pithos_init):
    """Setup account"""
104

105
    def __init__(self, arguments={}, auth_base=None, cloud=None):
106
        super(_pithos_account, self).__init__(arguments, auth_base, cloud)
107
108
109
        self['account'] = UserAccountArgument(
            'A user UUID or name', ('-A', '--account'))
        self.arguments['account'].account_client = auth_base
110

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    def print_objects(self, object_list):
        for index, obj in enumerate(object_list):
            pretty_obj = obj.copy()
            index += 1
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
            if 'subdir' in obj:
                continue
            if self._is_dir(obj):
                size = 'D'
            else:
                size = format_size(obj['bytes'])
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
            oname = obj['name'] if self['more'] else bold(obj['name'])
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
            if self['detail']:
                self.writeln('%s%s' % (prfx, oname))
                self.print_dict(pretty_obj, exclude=('name'))
                self.writeln()
            else:
                oname = '%s%9s %s' % (prfx, size, oname)
                oname += '/' if self._is_dir(obj) else u''
                self.writeln(oname)
133

134
135
    @staticmethod
    def _is_dir(remote_dict):
136
        return 'application/directory' in remote_dict.get(
137
            'content_type', remote_dict.get('content-type', ''))
138

139
140
141
142
    def _run(self):
        super(_pithos_account, self)._run()
        self.client.account = self['account'] or getattr(
            self, 'account', getattr(self.client, 'account', None))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
143

144

145
146
class _pithos_container(_pithos_account):
    """Setup container"""
147

148
    def __init__(self, arguments={}, auth_base=None, cloud=None):
149
        super(_pithos_container, self).__init__(arguments, auth_base, cloud)
150
        self['container'] = ValueArgument(
151
            'Use this container (default: pithos)', ('-C', '--container'))
152

153
154
    @staticmethod
    def _resolve_pithos_url(url):
155
156
157
        """Match urls of one of the following formats:
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
        /CONTAINER/OBJECT_PATH
158
        return account, container, path
159
        """
160
        account, container, obj_path, prefix = '', '', url, 'pithos://'
161
        if url.startswith(prefix):
162
            account, sep, url = url[len(prefix):].partition('/')
163
164
            url = '/%s' % url
        if url.startswith('/'):
165
166
            container, sep, obj_path = url[1:].partition('/')
        return account, container, obj_path
167

168
    def _run(self, url=None):
169
        acc, con, self.path = self._resolve_pithos_url(url or '')
170
        #  self.account = acc or getattr(self, 'account', '')
171
        super(_pithos_container, self)._run()
172
        self.container = con or self['container'] or getattr(
173
            self, 'container', None) or getattr(self.client, 'container', '')
174
        self.client.account = acc or self.client.account
175
        self.client.container = self.container
176

177

178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
@command(file_cmds)
class file_info(_pithos_container, _optional_json):
    """Get information/details about a file"""

    arguments = dict(
        object_version=ValueArgument(
            'download a file of a specific version', '--object-version'),
        hashmap=FlagArgument(
            'Get file hashmap instead of details', '--hashmap'),
        matching_etag=ValueArgument(
            'show output if ETags match', '--if-match'),
        non_matching_etag=ValueArgument(
            'show output if ETags DO NOT match', '--if-none-match'),
        modified_since_date=DateArgument(
            'show output modified since then', '--if-modified-since'),
        unmodified_since_date=DateArgument(
            'show output unmodified since then', '--if-unmodified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
195
196
197
198
199
        sharing=FlagArgument(
            'show object permissions and sharing information', '--sharing'),
        metadata=FlagArgument('show only object metadata', '--metadata'),
        versions=FlagArgument(
            'show the list of versions for the file', '--object-versions')
200
201
    )

Stavros Sachtouris's avatar
Stavros Sachtouris committed
202
203
204
205
206
    def version_print(self, versions):
        return {'/%s/%s' % (self.container, self.path): [
            dict(version_id=vitem[0], created=strftime(
                '%d-%m-%Y %H:%M:%S',
                localtime(float(vitem[1])))) for vitem in versions]}
207

208
    @errors.generic.all
209
210
211
212
213
214
215
216
217
218
219
220
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self):
        if self['hashmap']:
            r = self.client.get_object_hashmap(
                self.path,
                version=self['object_version'],
                if_match=self['matching_etag'],
                if_none_match=self['non_matching_etag'],
                if_modified_since=self['modified_since_date'],
                if_unmodified_since=self['unmodified_since_date'])
Stavros Sachtouris's avatar
Stavros Sachtouris committed
221
        elif self['sharing']:
222
            r = self.client.get_object_sharing(self.path)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
223
224
225
226
227
228
229
230
231
232
            r['public url'] = self.client.get_object_info(
                self.path, version=self['object_version']).get(
                    'x-object-public', None)
        elif self['metadata']:
            r, preflen = dict(), len('x-object-meta-')
            for k, v in self.client.get_object_meta(self.path).items():
                r[k[preflen:]] = v
        elif self['versions']:
            r = self.version_print(
                self.client.get_object_versionlist(self.path))
233
234
235
236
237
238
239
240
241
242
        else:
            r = self.client.get_object_info(
                self.path, version=self['object_version'])
        self._print(r, self.print_dict)

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()


243
244
245
@command(file_cmds)
class file_list(_pithos_container, _optional_json, _name_filter):
    """List all objects in a container or a directory object"""
246

Stavros Sachtouris's avatar
Stavros Sachtouris committed
247
    arguments = dict(
248
249
250
        detail=FlagArgument('detailed output', ('-l', '--list')),
        limit=IntArgument('limit number of listed items', ('-n', '--number')),
        marker=ValueArgument('output greater that marker', '--marker'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
251
252
        delimiter=ValueArgument('show output up to delimiter', '--delimiter'),
        meta=ValueArgument(
253
            'show output with specified meta keys', '--meta',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
254
255
            default=[]),
        if_modified_since=ValueArgument(
256
            'show output modified since then', '--if-modified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
257
        if_unmodified_since=ValueArgument(
258
            'show output not modified since then', '--if-unmodified-since'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
259
260
        until=DateArgument('show metadata until then', '--until'),
        format=ValueArgument(
261
            'format to parse until data (default: d/m/Y H:M:S )', '--format'),
262
263
264
        shared_by_me=FlagArgument(
            'show only files shared to other users', '--shared-by-me'),
        public=FlagArgument('show only published objects', '--public'),
265
        more=FlagArgument('read long results', '--more'),
266
267
268
269
        enum=FlagArgument('Enumerate results', '--enumerate'),
        recursive=FlagArgument(
            'Recursively list containers and their contents',
            ('-R', '--recursive'))
Stavros Sachtouris's avatar
Stavros Sachtouris committed
270
    )
271

272
273
274
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
275
    @errors.pithos.object_path
276
    def _run(self):
277
278
279
        r = self.client.container_get(
            limit=False if self['more'] else self['limit'],
            marker=self['marker'],
280
            prefix=self['name_pref'],
281
282
            delimiter=self['delimiter'],
            path=self.path or '',
283
            show_only_shared=self['shared_by_me'],
Stavros Sachtouris's avatar
Stavros Sachtouris committed
284
            public=self['public'],
285
286
287
            if_modified_since=self['if_modified_since'],
            if_unmodified_since=self['if_unmodified_since'],
            until=self['until'],
288
            meta=self['meta'])
289
290

        if not r.json:
291
            self.error('Container "%s" is empty' % self.client.container)
292

293
        files = self._filter_by_name(r.json)
294
295
296
        if self['more']:
            outbu, self._out = self._out, StringIO()
        try:
297
            if self['json_output'] or self['output_format']:
298
299
                self._print(files)
            else:
300
                self.print_objects(files)
301
302
303
304
        finally:
            if self['more']:
                pager(self._out.getvalue())
                self._out = outbu
305

306
    def main(self, path_or_url=''):
307
        super(self.__class__, self)._run(path_or_url)
308
        self._run()
309

Stavros Sachtouris's avatar
Stavros Sachtouris committed
310

311
312
313
314
315
316
317
318
319
320
321
322
323
@command(file_cmds)
class file_modify(_pithos_container):
    """Modify the attributes of a file or directory object"""

    arguments = dict(
        uuid_for_read_permission=RepeatableArgument(
            'Give read access to user/group (can be repeated, accumulative). '
            'Format for users: UUID . Format for groups: UUID:GROUP . '
            'Use * for all users/groups', '--read-permission'),
        uuid_for_write_permission=RepeatableArgument(
            'Give write access to user/group (can be repeated, accumulative). '
            'Format for users: UUID . Format for groups: UUID:GROUP . '
            'Use * for all users/groups', '--write-permission'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
324
325
326
327
328
329
        no_permissions=FlagArgument('Remove permissions', '--no-permissions'),
        metadata_to_set=KeyValueArgument(
            'Add metadata (KEY=VALUE) to an object (can be repeated)',
            '--metadata-add'),
        metadata_key_to_delete=RepeatableArgument(
            'Delete object metadata (can be repeated)', '--metadata-del'),
330
331
    )
    required = [
332
        'uuid_for_read_permission', 'metadata_to_set',
Stavros Sachtouris's avatar
Stavros Sachtouris committed
333
334
        'uuid_for_write_permission', 'no_permissions',
        'metadata_key_to_delete']
335

336
337
338
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
339
    @errors.pithos.object_path
340
    def _run(self):
341
342
343
344
345
346
        if self['uuid_for_read_permission'] or self[
                'uuid_for_write_permission']:
            perms = self.client.get_object_sharing(self.path)
            read, write = perms.get('read', ''), perms.get('write', '')
            read = read.split(',') if read else []
            write = write.split(',') if write else []
347
348
            read += (self['uuid_for_read_permission'] or [])
            write += (self['uuid_for_write_permission'] or [])
349
350
351
352
353
            self.client.set_object_sharing(
                self.path, read_permission=read, write_permission=write)
            self.print_dict(self.client.get_object_sharing(self.path))
        if self['no_permissions']:
            self.client.del_object_sharing(self.path)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
354
        metadata = self['metadata_to_set'] or dict()
355
        for k in (self['metadata_key_to_delete'] or []):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
356
357
358
359
            metadata[k] = ''
        if metadata:
            self.client.set_object_meta(self.path, metadata)
            self.print_dict(self.client.get_object_meta(self.path))
360
361
362
363
364
365
366

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        if self['no_permissions'] and (
                self['uuid_for_read_permission'] or self[
                    'uuid_for_write_permission']):
            raise CLIInvalidArgument(
367
368
                '%s cannot be used with other permission arguments' % (
                    self.arguments['no_permissions'].lvalue))
369
        self._run()
370

Stavros Sachtouris's avatar
Stavros Sachtouris committed
371

372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
@command(file_cmds)
class file_publish(_pithos_container):
    """Publish an object (creates a public URL)"""

    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self):
        self.writeln(self.client.publish_object(self.path))

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()


@command(file_cmds)
class file_unpublish(_pithos_container):
    """Unpublish an object"""

    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self):
        self.client.unpublish_object(self.path)

    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()


404
405
def _assert_path(self, path_or_url):
    if not self.path:
406
        raise CLIError(
407
408
409
410
            'Directory path is missing in location %s' % path_or_url,
            details=['Location format:    [[pithos://UUID]/CONTAINER/]PATH'])


411
412
@command(file_cmds)
class file_create(_pithos_container, _optional_output_cmd):
413
    """Create an empty file"""
414
415
416
417
418

    arguments = dict(
        content_type=ValueArgument(
            'Set content type (default: application/octet-stream)',
            '--content-type',
419
            default='application/octet-stream')
420
421
    )

422
423
424
425
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    def _run(self):
426
427
        self._optional_output(
            self.client.create_object(self.path, self['content_type']))
428

429
430
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
431
        _assert_path(self, path_or_url)
Stavros Sachtouris's avatar
Stavros Sachtouris committed
432
        self._run()
433
434
435
436


@command(file_cmds)
class file_mkdir(_pithos_container, _optional_output_cmd):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
437
    """Create a directory: /file create --content-type='application/directory'
438
439
440
441
442
    """

    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
443
    def _run(self, path):
444
445
        self._optional_output(self.client.create_directory(self.path))

Stavros Sachtouris's avatar
Stavros Sachtouris committed
446
447
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
448
449
        _assert_path(self, path_or_url)
        self._run(self.path)
450
451


452
@command(file_cmds)
453
class file_delete(_pithos_container):
454
    """Delete a file or directory object"""
455

Stavros Sachtouris's avatar
Stavros Sachtouris committed
456
    arguments = dict(
457
458
459
460
461
462
        until_date=DateArgument('remove history until then', '--until'),
        yes=FlagArgument('Do not prompt for permission', '--yes'),
        recursive=FlagArgument(
            'If a directory, empty first', ('-r', '--recursive')),
        delimiter=ValueArgument(
            'delete objects prefixed with <object><delimiter>', '--delimiter')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
463
    )
464

465
466
467
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
468
469
470
471
472
    @errors.pithos.object_path
    def _run(self):
        if self.path:
            if self['yes'] or self.ask_user(
                    'Delete /%s/%s ?' % (self.container, self.path)):
473
                self.client.del_object(
474
475
                    self.path,
                    until=self['until_date'],
476
                    delimiter='/' if self['recursive'] else self['delimiter'])
477
478
479
            else:
                self.error('Aborted')
        else:
480
481
482
483
484
            if self['yes'] or self.ask_user(
                    'Empty container /%s ?' % self.container):
                self.client.container_delete(self.container, delimiter='/')
            else:
                self.error('Aborted')
485

486
487
488
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run()
489

Stavros Sachtouris's avatar
Stavros Sachtouris committed
490

491
class _source_destination(_pithos_container, _optional_output_cmd):
Stavros Sachtouris's avatar
Stavros Sachtouris committed
492

493
    sd_arguments = dict(
494
495
        destination_user=UserAccountArgument(
            'UUID or username, default: current user', '--to-account'),
496
497
498
499
500
501
        destination_container=ValueArgument(
            'default: pithos', '--to-container'),
        source_prefix=FlagArgument(
            'Transfer all files that are prefixed with SOURCE PATH If the '
            'destination path is specified, replace SOURCE_PATH with '
            'DESTINATION_PATH',
502
            ('-r', '--recursive')),
503
        force=FlagArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
504
505
506
            'Overwrite destination objects, if needed', ('-f', '--force')),
        source_version=ValueArgument(
            'The version of the source object', '--source-version')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
507
    )
508

509
    def __init__(self, arguments={}, auth_base=None, cloud=None):
510
        self.arguments.update(arguments)
511
512
        self.arguments.update(self.sd_arguments)
        super(_source_destination, self).__init__(
513
            self.arguments, auth_base, cloud)
514
        self.arguments['destination_user'].account_client = self.auth_base
515

516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
    def _report_transfer(self, src, dst, transfer_name):
        if not dst:
            if transfer_name in ('move', ):
                self.error('  delete source directory %s' % src)
            return
        dst_prf = '' if self.account == self.dst_client.account else (
                'pithos://%s' % self.dst_client.account)
        if src:
            src_prf = '' if self.account == self.dst_client.account else (
                    'pithos://%s' % self.account)
            self.error('  %s %s/%s/%s\n  -->  %s/%s/%s' % (
                transfer_name,
                src_prf, self.container, src,
                dst_prf, self.dst_client.container, dst))
        else:
            self.error('  mkdir %s/%s/%s' % (
                dst_prf, self.dst_client.container, dst))
533
534
535

    @errors.generic.all
    @errors.pithos.account
536
537
538
539
540
541
542
    def _src_dst(self, version=None):
        """Preconditions:
        self.account, self.container, self.path
        self.dst_acc, self.dst_con, self.dst_path
        They should all be configured properly
        :returns: [(src_path, dst_path), ...], if src_path is None, create
            destination directory
543
        """
544
        src_objects, dst_objects, pairs = dict(), dict(), []
545
        try:
546
547
548
549
550
551
552
553
            for obj in self.dst_client.list_objects(
                    prefix=self.dst_path or self.path or '/'):
                dst_objects[obj['name']] = obj
        except ClientError as ce:
            if ce.status in (404, ):
                raise CLIError(
                    'Destination container pithos://%s/%s not found' % (
                        self.dst_client.account, self.dst_client.container))
554
            raise ce
555
556
        if self['source_prefix']:
            #  Copy and replace prefixes
557
            for src_obj in self.client.list_objects(prefix=self.path):
558
559
560
561
562
563
564
                src_objects[src_obj['name']] = src_obj
            for src_path, src_obj in src_objects.items():
                dst_path = '%s%s' % (
                    self.dst_path or self.path, src_path[len(self.path):])
                dst_obj = dst_objects.get(dst_path, None)
                if self['force'] or not dst_obj:
                    #  Just do it
565
566
567
568
                    pairs.append((
                        None if self._is_dir(src_obj) else src_path, dst_path))
                    if self._is_dir(src_obj):
                        pairs.append((self.path or dst_path, None))
569
570
571
572
573
574
575
576
577
578
579
580
                elif not (self._is_dir(dst_obj) and self._is_dir(src_obj)):
                    raise CLIError(
                        'Destination object exists', importance=2, details=[
                            'Failed while transfering:',
                            '    pithos://%s/%s/%s' % (
                                    self.account,
                                    self.container,
                                    src_path),
                            '--> pithos://%s/%s/%s' % (
                                    self.dst_client.account,
                                    self.dst_client.container,
                                    dst_path),
581
582
                            'Use %s to transfer overwrite' % (
                                    self.arguments['force'].lvalue)])
583
        else:
584
585
            #  One object transfer
            try:
Stavros Sachtouris's avatar
Stavros Sachtouris committed
586
587
588
589
                src_version_arg = self.arguments.get('source_version', None)
                src_obj = self.client.get_object_info(
                    self.path,
                    version=src_version_arg.value if src_version_arg else None)
590
591
592
593
594
595
            except ClientError as ce:
                if ce.status in (204, ):
                    raise CLIError(
                        'Missing specific path container %s' % self.container,
                        importance=2, details=[
                            'To transfer container contents %s' % (
596
                                self.arguments['source_prefix'].lvalue)])
597
                raise
598
            dst_path = self.dst_path or self.path
Stavros Sachtouris's avatar
Stavros Sachtouris committed
599
            dst_obj = dst_objects.get(dst_path or self.path, None)
600
601
            if self['force'] or not dst_obj:
                pairs.append(
602
603
604
                    (None if self._is_dir(src_obj) else self.path, dst_path))
                if self._is_dir(src_obj):
                    pairs.append((self.path or dst_path, None))
605
606
607
608
609
610
611
            elif self._is_dir(src_obj):
                raise CLIError(
                    'Cannot transfer an application/directory object',
                    importance=2, details=[
                        'The object pithos://%s/%s/%s is a directory' % (
                            self.account,
                            self.container,
612
                            self.path),
613
                        'To recursively copy a directory, use',
614
                        '  %s' % self.arguments['source_prefix'].lvalue,
615
616
617
618
619
620
621
622
623
624
625
                        'To create a file, use',
                        '  /file create  (general purpose)',
                        '  /file mkdir   (a directory object)'])
            else:
                raise CLIError(
                    'Destination object exists',
                    importance=2, details=[
                        'Failed while transfering:',
                        '    pithos://%s/%s/%s' % (
                                self.account,
                                self.container,
626
                                self.path),
627
628
629
630
                        '--> pithos://%s/%s/%s' % (
                                self.dst_client.account,
                                self.dst_client.container,
                                dst_path),
631
632
                        'Use %s to transfer overwrite' % (
                                self.arguments['force'].lvalue)])
633
634
635
636
637
638
639
640
641
642
        return pairs

    def _run(self, source_path_or_url, destination_path_or_url=''):
        super(_source_destination, self)._run(source_path_or_url)
        dst_acc, dst_con, dst_path = self._resolve_pithos_url(
            destination_path_or_url)
        self.dst_client = PithosClient(
            base_url=self.client.base_url, token=self.client.token,
            container=self[
                'destination_container'] or dst_con or self.client.container,
643
            account=self['destination_user'] or dst_acc or self.account)
644
        self.dst_path = dst_path or self.path
645

646

647
648
649
@command(file_cmds)
class file_copy(_source_destination):
    """Copy objects, even between different accounts or containers"""
650
651

    arguments = dict(
652
        public=ValueArgument('publish new object', '--public'),
653
        content_type=ValueArgument(
654
            'change object\'s content type', '--content-type'),
655
        source_version=ValueArgument(
Stavros Sachtouris's avatar
Stavros Sachtouris committed
656
            'The version of the source object', '--object-version')
657
658
    )

659
660
661
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
662
    @errors.pithos.account
663
664
    def _run(self):
        for src, dst in self._src_dst(self['source_version']):
665
666
            self._report_transfer(src, dst, 'copy')
            if src and dst:
667
668
669
670
671
                self.dst_client.copy_object(
                    src_container=self.client.container,
                    src_object=src,
                    dst_container=self.dst_client.container,
                    dst_object=dst,
672
                    source_account=self.client.account,
673
674
675
                    source_version=self['source_version'],
                    public=self['public'],
                    content_type=self['content_type'])
676
677
            elif dst:
                self.dst_client.create_directory(dst)
678
679

    def main(self, source_path_or_url, destination_path_or_url=None):
680
        super(file_copy, self)._run(
681
682
683
684
685
686
687
            source_path_or_url, destination_path_or_url or '')
        self._run()


@command(file_cmds)
class file_move(_source_destination):
    """Move objects, even between different accounts or containers"""
Stavros Sachtouris's avatar
Stavros Sachtouris committed
688
689

    arguments = dict(
690
        public=ValueArgument('publish new object', '--public'),
691
        content_type=ValueArgument(
692
            'change object\'s content type', '--content-type')
Stavros Sachtouris's avatar
Stavros Sachtouris committed
693
    )
694

695
696
697
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
698
699
700
    @errors.pithos.account
    def _run(self):
        for src, dst in self._src_dst():
701
702
            self._report_transfer(src, dst, 'move')
            if src and dst:
703
704
705
706
707
708
709
710
                self.dst_client.move_object(
                    src_container=self.client.container,
                    src_object=src,
                    dst_container=self.dst_client.container,
                    dst_object=dst,
                    source_account=self.account,
                    public=self['public'],
                    content_type=self['content_type'])
711
712
            elif dst:
                self.dst_client.create_directory(dst)
713
            else:
714
                self.client.del_object(src)
715
716
717
718
719

    def main(self, source_path_or_url, destination_path_or_url=None):
        super(file_move, self)._run(
            source_path_or_url, destination_path_or_url or '')
        self._run()
720
721
722
723


@command(file_cmds)
class file_append(_pithos_container, _optional_output_cmd):
724
725
726
727
728
    """Append local file to (existing) remote object
    The remote object should exist.
    If the remote object is a directory, it is transformed into a file.
    In the later case, objects under the directory remain intact.
    """
729

Stavros Sachtouris's avatar
Stavros Sachtouris committed
730
731
    arguments = dict(
        progress_bar=ProgressBarArgument(
732
            'do not show progress bar', ('-N', '--no-progress-bar'),
733
734
            default=False),
        max_threads=IntArgument('default: 1', '--threads'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
735
    )
736

737
738
739
740
741
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    def _run(self, local_path):
742
743
        if self['max_threads'] > 0:
            self.client.MAX_THREADS = int(self['max_threads'])
744
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
745
        try:
746
747
748
            with open(local_path, 'rb') as f:
                self._optional_output(
                    self.client.append_object(self.path, f, upload_cb))
749
        finally:
750
751
            self._safe_progress_bar_finish(progress_bar)

752
753
    def main(self, local_path, remote_path_or_url):
        super(self.__class__, self)._run(remote_path_or_url)
754
        self._run(local_path)
755

Stavros Sachtouris's avatar
Stavros Sachtouris committed
756

757
758
759
760
761
762
763
764
@command(file_cmds)
class file_truncate(_pithos_container, _optional_output_cmd):
    """Truncate remote file up to size"""

    arguments = dict(
        size_in_bytes=IntArgument('Length of file after truncation', '--size')
    )
    required = ('size_in_bytes', )
765

766
767
768
769
770
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
771
    def _run(self, size):
772
        self._optional_output(self.client.truncate_object(self.path, size))
773

774
775
776
    def main(self, path_or_url):
        super(self.__class__, self)._run(path_or_url)
        self._run(size=self['size_in_bytes'])
777

Stavros Sachtouris's avatar
Stavros Sachtouris committed
778

779
780
781
@command(file_cmds)
class file_overwrite(_pithos_container, _optional_output_cmd):
    """Overwrite part of a remote file"""
782

Stavros Sachtouris's avatar
Stavros Sachtouris committed
783
784
    arguments = dict(
        progress_bar=ProgressBarArgument(
785
            'do not show progress bar', ('-N', '--no-progress-bar'),
786
787
            default=False),
        start_position=IntArgument('File position in bytes', '--from'),
788
        end_position=IntArgument('File position in bytes', '--to'),
789
        object_version=ValueArgument('File to overwrite', '--object-version'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
790
    )
791
    required = ('start_position', 'end_position')
792

793
794
795
796
797
798
    @errors.generic.all
    @errors.pithos.connection
    @errors.pithos.container
    @errors.pithos.object_path
    @errors.pithos.object_size
    def _run(self, local_path, start, end):
799
        start, end = int(start), int(end)
800
801
        (progress_bar, upload_cb) = self._safe_progress_bar(
            'Overwrite %s bytes' % (end - start))
802
        try:
803
804
805
806
807
808
            with open(path.abspath(local_path), 'rb') as f:
                self._optional_output(self.client.overwrite_object(
                    obj=self.path,
                    start=start,
                    end=end,
                    source_file=f,
809
                    source_version=self['object_version'],
810
                    upload_cb=upload_cb))
811
        finally:
812
813
            self._safe_progress_bar_finish(progress_bar)

814
815
    def main(self, local_path, path_or_url):
        super(self.__class__, self)._run(path_or_url)
816
        self.path = self.path or path.basename(local_path)
817
818
819
820
        self._run(
            local_path=local_path,
            start=self['start_position'],
            end=self['end_position'])
821

822

823
824
@command(file_cmds)
class file_upload(_pithos_container, _optional_output_cmd):
825
826
    """Upload a file"""

Stavros Sachtouris's avatar
Stavros Sachtouris committed
827
    arguments = dict(
828
        max_threads=IntArgument('default: 5', '--threads'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
829
        content_encoding=ValueArgument(
830
            'set MIME content type', '--content-encoding'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
831
        content_disposition=ValueArgument(
832
            'specify objects presentation style', '--content-disposition'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
833
        content_type=ValueArgument('specify content type', '--content-type'),
834
        uuid_for_read_permission=RepeatableArgument(
835
836
            'Give read access to a user or group (can be repeated) '
            'Use * for all users',
837
838
            '--read-permission'),
        uuid_for_write_permission=RepeatableArgument(
839
840
            'Give write access to a user or group (can be repeated) '
            'Use * for all users',
841
            '--write-permission'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
842
843
844
        public=FlagArgument('make object publicly accessible', '--public'),
        progress_bar=ProgressBarArgument(
            'do not show progress bar',
845
            ('-N', '--no-progress-bar'),
846
            default=False),
847
848
849
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
        recursive=FlagArgument(
            'Recursively upload directory *contents* + subdirectories',
850
851
852
853
854
855
856
            ('-r', '--recursive')),
        unchunked=FlagArgument(
            'Upload file as one block (not recommended)', '--unchunked'),
        md5_checksum=ValueArgument(
            'Confirm upload with a custom checksum (MD5)', '--etag'),
        use_hashes=FlagArgument(
            'Source file contains hashmap not data', '--source-is-hashmap'),
Stavros Sachtouris's avatar
Stavros Sachtouris committed
857
    )
858

859
860
861
862
863
864
865
866
867
868
    def _sharing(self):
        sharing = dict()
        readlist = self['uuid_for_read_permission']
        if readlist:
            sharing['read'] = self['uuid_for_read_permission']
        writelist = self['uuid_for_write_permission']
        if writelist:
            sharing['write'] = self['uuid_for_write_permission']
        return sharing or None

869
870
871
    def _check_container_limit(self, path):
        cl_dict = self.client.get_container_limit()
        container_limit = int(cl_dict['x-container-policy-quota'])
872
873
        r = self.client.container_get()
        used_bytes = sum(int(o['bytes']) for o in r.json)
874
        path_size = get_path_size(path)
875
        if container_limit and path_size > (container_limit - used_bytes):
876
877
            raise CLIError(
                'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
878
879
                    self.client.container,
                    format_size(container_limit),
880
                    format_size(used_bytes),
881
882
                    format_size(path_size),
                    path),
883
                details=[
884
885
886
887
888
889
890
                    'Check accound limit: /file quota',
                    'Check container limit:',
                    '\t/file containerlimit get %s' % self.client.container,
                    'Increase container limit:',
                    '\t/file containerlimit set <new limit> %s' % (
                        self.client.container)])

891
    def _src_dst(self, local_path, remote_path, objlist=None):
892
        lpath = path.abspath(local_path)
893
        short_path = path.basename(path.abspath(local_path))
894
895
        rpath = remote_path or short_path
        if path.isdir(lpath):
896
            if not self['recursive']:
897
                raise CLIError('%s is a directory' % lpath, details=[
898
899
                    'Use %s to upload directories & contents' % (
                        self.arguments['recursive'].lvalue)])
900
901
            robj = self.client.container_get(path=rpath)
            if not self['overwrite']:
902
903
904
                if robj.json:
                    raise CLIError(
                        'Objects/files prefixed as %s already exist' % rpath,
905
                        details=['Existing objects:'] + ['\t/%s/\t%s' % (
906
907
908
909
910
911
912
913
914
915
916
917
918
919
                            o['name'],
                            o['content_type'][12:]) for o in robj.json] + [
                            'Use -f to add, overwrite or resume'])
                else:
                    try:
                        topobj = self.client.get_object_info(rpath)
                        if not self._is_dir(topobj):
                            raise CLIError(
                                'Object /%s/%s exists but not a directory' % (
                                    self.container, rpath),
                                details=['Use -f to overwrite'])
                    except ClientError as ce:
                        if ce.status not in (404, ):
                            raise
920
            self._check_container_limit(lpath)
921
922
923
924
925
926
927
928
            prev = ''
            for top, subdirs, files in walk(lpath):
                if top != prev:
                    prev = top
                    try:
                        rel_path = rpath + top.split(lpath)[1]
                    except IndexError:
                        rel_path = rpath
929
                    self.error('mkdir /%s/%s' % (
930
                        self.client.container, rel_path))
931
932
933
934
                    self.client.create_directory(rel_path)
                for f in files:
                    fpath = path.join(top, f)
                    if path.isfile(fpath):
935
936
937
                        rel_path = rel_path.replace(path.sep, '/')
                        pathfix = f.replace(path.sep, '/')
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
938
                    else:
939
                        self.error('%s is not a regular file' % fpath)
940
        else:
941
            if not path.isfile(lpath):
942
                raise CLIError(('%s is not a regular file' % lpath) if (
943
                    path.exists(lpath)) else '%s does not exist' % lpath)
944
945
946
            try:
                robj = self.client.get_object_info(rpath)
                if remote_path and self._is_dir(robj):
947
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
948
949
                    self.client.get_object_info(rpath)
                if not self['overwrite']:
950
951
952
953
                    raise CLIError(
                        'Object /%s/%s already exists' % (
                            self.container, rpath),
                        details=['use -f to overwrite / resume'])
954
            except ClientError as ce:
955
                if ce.status not in (404, ):
956
                    raise
957
            self._check_container_limit(lpath)
958
            yield open(lpath, 'rb'), rpath
959

960
    def _run(self, local_path, remote_path):
961
        self.client.MAX_THREADS = int(self['max_threads'] or 5)
962
963
        params = dict(
            content_encoding=self['content_encoding'],
Stavros Sachtouris's avatar
Stavros Sachtouris committed
964
965
            content_type=self['content_type'],
            content_disposition=self['content_disposition'],