Commit 03a41be8 authored by Nikos Skalkotos's avatar Nikos Skalkotos

Merge branch 'master' into develop

Bump version to 0.13.2next
Conflicts:
	kamaki/version.py
	version
parents ce8d24ba f8e37233
......@@ -23,6 +23,17 @@ Features
.. _Changelog-0.13:
v0.13.2
=======
Bug Fixes
---------
* Fix delimiter handling when uploading/downloading in windows systems
* Introduce a new command "kamaki scripts verifyfs" to verify and
repair affected containers
* Create subnets without gateway IPs
v0.13.1
=======
......
......@@ -372,6 +372,7 @@ class subnet_create(_NetworkInit, OptionalOutput):
' e.g., --alloc-pool=123.45.67.1,123.45.67.8',
'--alloc-pool'),
gateway=ValueArgument('Gateway IP', '--gateway'),
no_gateway=FlagArgument('Do not assign a gateway IP', '--no-gateway'),
subnet_id=ValueArgument('The id for the subnet', '--id'),
ipv6=FlagArgument('If set, IP version is set to 6, else 4', '--ipv6'),
enable_dhcp=FlagArgument('Enable dhcp (default: off)', '--with-dhcp'),
......@@ -383,10 +384,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
@errors.Generic.all
@errors.Cyclades.connection
def _run(self):
gateway = '' if self['no_gateway'] else self['gateway']
try:
net = self.client.create_subnet(
self['network_id'], self['cidr'],
self['name'], self['allocation_pools'], self['gateway'],
self['name'], self['allocation_pools'], gateway,
self['subnet_id'], self['ipv6'], self['enable_dhcp'])
except ClientError as ce:
if ce.status in (404, 400):
......@@ -396,6 +398,11 @@ class subnet_create(_NetworkInit, OptionalOutput):
def main(self):
super(self.__class__, self)._run()
if self['gateway'] and self['no_gateway']:
raise CLIInvalidArgument('Conflicting arguments', details=[
'Arguments %s and %s cannot be used together' % (
self.arguments['gateway'].lvalue,
self.arguments['no_gateway'].lvalue)])
self._run()
......
......@@ -519,10 +519,31 @@ class file_delete(_PithosContainer):
self.client.get_object_info(self.path)
if self['yes'] or self.ask_user(
'Delete /%s/%s ?' % (self.container, self.path)):
self.client.del_object(
self.path,
until=self['until_date'],
delimiter='/' if self['recursive'] else self['delimiter'])
# See if any objects exist under prefix
# Add a trailing / to object's name
prefix = self.path.rstrip('/') + '/'
result = self.client.container_get(prefix=prefix)
if result.json:
count = len(result.json)
self.error(' * %d other object(s) with %s as prefix found' %
(count, prefix))
if self['recursive']:
msg = 'The above %d object(s) will be deleted, too' % \
count
else:
msg = 'The above %d object(s) will be preserved,' \
' but the directory structure' \
' will become inconsistent' % count
self.error(' * %s!' % msg)
if not result.json or self.ask_user("Continue?"):
self.client.del_object(
self.path,
until=self['until_date'],
delimiter='/' if self['recursive'] else self['delimiter'])
else:
self.error('Aborted')
......@@ -998,7 +1019,10 @@ class file_upload(_PithosContainer):
rel_path = rpath + top.split(lpath)[1]
except IndexError:
rel_path = rpath
self.error('mkdir /%s/%s' % (
# Use the '/' separator for directories that
# are about to be created in Pithos
rel_path = rel_path.replace(path.sep, '/')
self.error('remote: mkdir /%s/%s' % (
self.client.container, rel_path))
self.client.create_directory(rel_path)
for f in files:
......@@ -1221,11 +1245,16 @@ class file_download(_PithosContainer):
and they are pretended to other objects in a very strict order (shorter
to longer path)."""
ret, obj = [], None
# The prefix is actually the relative remote path without
# the trailing separator.
prefix = self.path.rstrip('/')
try:
if self.path:
# prefix here is the object's path we requested to download
if prefix:
obj = self.client.get_object_info(
self.path, version=self['object_version'])
obj.setdefault('name', self.path.strip('/'))
prefix, version=self['object_version'])
#FIXME: why is this needed????
obj.setdefault('name', prefix)
except ClientError as ce:
if ce.status in (404, ):
self._container_exists()
......@@ -1246,29 +1275,37 @@ class file_download(_PithosContainer):
' kamaki container download %s [LOCAL_PATH]' % (
self.container)])
raise
rpath = self.path.strip('/')
if local_path and self.path and local_path.endswith('/'):
local_path = local_path.rstrip('/') or '/'
# We requested to download either a whole container or a directory
if (not obj) or self.object_is_dir(obj):
if self['recursive']:
if not (self.path or local_path.endswith('/')):
# Download the whole container
local_path = '' if local_path in ('.', ) else local_path
local_path = '%s/' % (local_path or self.container)
obj = obj or dict(
name='', content_type='application/directory')
dirs, files = [], []
objects = self.client.container_get(
prefix=rpath,
result = self.client.container_get(
prefix=prefix,
if_modified_since=self['modified_since_date'],
if_unmodified_since=self['unmodified_since_date'])
for o in objects.json:
(dirs if self.object_is_dir(o) else files).append(o)
# Find the final local path for each remote object
# [(remote name, final local path),.]
for o in result.json:
remote = o['name']
# First find the relative path of the object
# without the prefix and any leading '/'
relative = remote[len(prefix):].lstrip('/')
# Translate it to a valid path with proper separator
norm = relative.replace('/', path.sep)
# Append it to the desired local path
final = path.join(local_path, norm)
if self.object_is_dir(o):
dirs.append((remote, final))
else:
files.append((remote, final))
self.error(r"%s -> %s" % (remote, final))
# Put the directories on top of the list
for dpath in sorted(['%s%s' % (
local_path, d['name'][len(rpath):]) for d in dirs]):
for dpath in sorted(p for _, p in dirs):
if path.exists(dpath):
if path.isdir(dpath):
continue
......@@ -1281,13 +1318,12 @@ class file_download(_PithosContainer):
ret.append((None, dpath, None))
# Append the file objects
for opath in [o['name'] for o in files]:
lpath = '%s%s' % (local_path, opath[len(rpath):])
for opath, lpath in files:
if self['resume']:
fxists = path.exists(lpath)
if fxists and path.isdir(lpath):
raise CLIError(
'Cannot change local dir %s info file' % (
'Cannot change local dir %s into a file' % (
lpath),
details=[
'Either remove the file or specify a'
......@@ -1300,10 +1336,10 @@ class file_download(_PithosContainer):
self.arguments['resume'].lvalue)])
else:
ret.append((opath, lpath, None))
elif self.path:
elif prefix:
raise CLIError(
'Remote object /%s/%s is a directory' % (
self.container, local_path),
self.container, prefix),
details=['Use %s to download directories' % (
self.arguments['recursive'].lvalue)])
else:
......@@ -1316,33 +1352,25 @@ class file_download(_PithosContainer):
parsed_name, self.container)])
else:
# Remote object is just a file
# The local path to be stored already exists
if path.exists(local_path):
if not self['resume']:
raise CLIError(
'Cannot overwrite local file %s' % (local_path),
details=['To overwrite/resume, use %s' % (
self.arguments['resume'].lvalue)])
elif '/' in local_path[1:-1]:
dirs = [p for p in local_path.split('/') if p]
pref = '/' if local_path.startswith('/') else ''
for d in dirs[:-1]:
pref += d
if not path.exists(pref):
ret.append((None, d, None))
elif not path.isdir(pref):
raise CLIError(
'Failed to use %s as a destination' % local_path,
importance=3,
details=[
'Local file %s is not a directory' % pref,
'Destination prefix must consist of '
'directories or non-existing names',
'Either remove the file, or choose another '
'destination'])
ret.append((rpath, local_path, self['resume']))
# The local path does not exist.
elif path.sep in local_path:
# Delegate intermediate local dir cration
# to makedirs() inside _run()
d = path.dirname(local_path)
ret.append((None, d, None))
ret.append((prefix, local_path, self['resume']))
for r, l, resume in ret:
if r:
with open(l, 'rwb+' if resume else 'wb+') as f:
mode = 'rb+' if resume and path.exists(l) else 'wb+'
with open(l, mode) as f:
yield (r, f)
else:
yield (r, l)
......@@ -1357,11 +1385,17 @@ class file_download(_PithosContainer):
self.client.MAX_THREADS = int(self['max_threads'] or 5)
progress_bar = None
try:
# From _src_dst():
# If rpath is None output_file is a directory.
# If rpath is not None output_file is a file descriptor.
for rpath, output_file in self._src_dst(local_path):
# Create a directory
if not rpath:
self.error('Create local directory %s' % output_file)
makedirs(output_file)
if not path.exists(output_file):
self.error('Create local directory %s' % output_file)
makedirs(output_file)
continue
# Download a file
self.error('/%s/%s --> %s' % (
self.container, rpath, output_file.name))
progress_bar, download_cb = self._safe_progress_bar(
......@@ -1398,14 +1432,19 @@ class file_download(_PithosContainer):
self.error('Download completed')
def main(self, remote_path_or_url, local_path=None):
""" Dowload remote_path_or_url to local_path. """
super(self.__class__, self)._run(remote_path_or_url)
local_path, rpath = local_path or '.', self.path or self.container
if local_path.endswith('.'):
local_path = local_path[:-1] + path.basename(self.path.rstrip('/'))
elif local_path.endswith(path.sep):
local_path += path.basename(rpath.rstrip('/'))
# Translate relative remote path to local path with proper separator
# and without trailing '/'. If not given use the name of the container
rpath = self.path.rstrip('/').replace('/', path.sep) or self.container
# If remote path is /pithos/dir1/dir2/ then here we download dir2
base = path.basename(rpath)
# If local_path is not given use current dir
if not local_path:
local_path = path.join('.', base)
# existing_dir/ -> existing_dir/base
elif path.exists(local_path) and path.isdir(local_path):
local_path += path.sep + path.basename(rpath.rstrip('/'))
local_path = path.join(local_path, base)
self._run(local_path=local_path)
......
......@@ -168,6 +168,7 @@ DEFAULTS = {
'config_cli': 'config',
'history_cli': 'history',
'ignore_ssl': 'off',
'scripts_cli': 'contrib.scripts',
'ca_certs': CACERTS_DEFAULT_PATH,
# Optional command specs:
# 'service_cli': 'astakos'
......
# Copyright 2015 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
# Copyright 2015 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
from datetime import date
from kamaki.cli import command
from kamaki.cli.errors import CLIError
from kamaki.cli.cmdtree import CommandTree
from kamaki.cli.cmds import errors, OptionalOutput
from kamaki.cli.cmds.pithos import _PithosAccount
from kamaki.cli.argument import FlagArgument
scripts_cmds = CommandTree('scripts', 'Useful scripts')
namespaces = [scripts_cmds, ]
@command(scripts_cmds)
class scripts_verifyfs(_PithosAccount, OptionalOutput):
"""Verify/Fix the structure of directory objects inside a container"""
arguments = dict(
fix_conflicts=FlagArgument(
'Fix conflicting names by renaming them '
'(prepare the structure of directory objects to be consistent)',
'--fix-conflicts'),
fix_names=FlagArgument(
'Rename directory objects containing \\',
'--fix-dir-names'),
fix_missing=FlagArgument(
'Create missing directories objects',
'--fix-missing-dirs'),
yes=FlagArgument('Do not prompt for permission', '--yes'),
)
@errors.Generic.all
@errors.Pithos.connection
@errors.Pithos.container
def _run(self):
dirs, files, empty_files = [], [], []
result = self.client.container_get()
for o in result.json:
name = o['name']
if self.object_is_dir(o):
dirs.append(name)
elif o['bytes'] == 0:
empty_files.append(name)
else:
files.append(name)
# Find all directories with backslashes
wrong = set()
for d in dirs:
if '\\' in d:
wrong.add(d)
# Find all intermediate directories and see if a missing directory
# exists or if an intermediate directory conflicts with an existing
# object name
missing = set()
conflicts = set()
for n in files + dirs:
inter = n.split('/')
inter.pop()
d = []
for i in inter:
d.append(i)
p = '/'.join(d)
if p not in dirs:
missing.add(p)
if p in files + empty_files:
conflicts.add(p)
# First try to resolve conflicts
if self['fix_conflicts']:
for c in conflicts:
if self['yes'] or self.ask_user('Rename %s?' % c):
backup = '%s_orig_%s' % (c, date.today().isoformat())
# TODO: check if backup name already exists
self.error(' * Renaming %s to %s' % (c, backup))
self.client.move_object(
src_container=self.client.container,
src_object=c,
dst_container=self.client.container,
dst_object=backup)
elif conflicts:
raise CLIError(
'Conflicting object names found: %s' % conflicts,
details=[
'They should be directory objects instead',
'Use --fix-conflicts to rename them and prepare'
' the directory structure for further fix actions'])
# renames should take place after fixing conflicts
elif self['fix_names']:
for w in wrong:
correct = w.replace('\\', '/')
if self['yes'] or self.ask_user('Rename %s?' % w):
self.error(' * Renaming %s to %s' % (w, correct))
self.client.move_object(
src_container=self.client.container,
src_object=w,
dst_container=self.client.container,
dst_object=correct)
elif wrong:
raise CLIError(
'Directory objects with backslashes found: %s' % wrong,
details=['Use --fix-dir-names to sanitize them'])
# missing dirs should be created after fixing names
elif self['fix_missing']:
for d in missing:
if self['yes'] or self.ask_user('Create %s?' % d):
self.error(' * Creating directory object %s' % d)
self.client.create_directory(d)
elif missing:
raise CLIError(
'Missing directory objects found: %s' % missing,
details=['Use --fix-missing-dirs to create them'])
def main(self, container):
super(self.__class__, self)._run()
self.container, self.client.container = container, container
self._run()
......@@ -387,8 +387,10 @@ def ask_user(msg, true_resp=('y', ), **kwargs):
msg = escape_ctrl_chars(msg).encode(pref_enc, 'replace')
yep = yep.encode(pref_enc, 'replace')
nope = nope.encode(pref_enc, 'replace')
user_response = raw_input(
response = raw_input(
'%s [%s/%s]: ' % (msg, yep, nope))
# Pressing just enter gives an empty response!
user_response = response if response else 'N'
return user_response[0].lower() in [s.lower() for s in true_resp]
......
......@@ -123,8 +123,9 @@ class NetworkClient(NetworkRestClient, Waiter):
:param name: (str) The subnet name
:param allocation_pools: (list of dicts) start/end addresses of
allocation pools: [{'start': ..., 'end': ...}, ...]
:param gateway_ip: (str)
:param subnet_id: (str)
:param gateway_ip: (str) Special cases:
None: server applies the default policy
empty iterable: no gateway IP on this subnet
:param ipv6: (bool) ip_version == 6 if true else 4 (default)
:param enable_dhcp: (bool)
"""
......@@ -134,8 +135,8 @@ class NetworkClient(NetworkRestClient, Waiter):
subnet['name'] = name
if allocation_pools:
subnet['allocation_pools'] = allocation_pools
if gateway_ip:
subnet['gateway_ip'] = gateway_ip
if gateway_ip is not None:
subnet['gateway_ip'] = gateway_ip or None
if subnet_id:
subnet['id'] = subnet_id
if enable_dhcp not in (None, ):
......
__version__ = "0.13.1next"
__version__ = "0.13.2next"
__version_vcs_info__ = {
'branch': 'feature-wait',
'revid': '68cc68c',
'revno': 2492}
__version_user_email__ = "saxtouri@admin.grnet.gr"
__version_user_name__ = "Stavros Sachtouris"
'branch': 'develop',
'revid': 'ce8d24b',
'revno': 2506}
__version_user_email__ = "skalkoto@grnet.gr"
__version_user_name__ = "Nikos Skalkotos"
#!/usr/bin/env python
# Copyright 2011-2013 GRNET S.A. All rights reserved.
# Copyright 2011-2015 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
......@@ -67,6 +67,7 @@ setup(
'kamaki.cli.argument',
'kamaki.cli.cmds',
'kamaki.cli.cmdtree',
'kamaki.cli.contrib',
'kamaki.clients',
'kamaki.clients.utils',
'kamaki.clients.astakos',
......
0.13.1next
0.13.2next
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment