Commit 4b609746 authored by Dimitris Aragiorgis's avatar Dimitris Aragiorgis
Browse files

scripts: Verify/Fix remote directory structure

A combination of kamaki and Pithos UI usage can lead to a
Pithos state that is inconsistent and does not match
ordinary directory structures.

Specifically, if we have a file under .\dir1\dir2\dir3 and
we upload dir1 recursively from a windows machine, pithos
objects will look like this:

    D dir1/
  10B dir1/dir2/dir3/file
    D dir1\dir2
    D dir1\dir2\dir3

If we traverse the above file structure from Pithos UI two
additional files will be created:

   0B dir1/dir2
   0B dir1/dir2/dir3

The above is a buggy behavior.

This patch introduces the 'kamaki scripts verifyfs' command.
It fixes the above structure in the following manner:

 a) --fix-conflicts will prepare the directory structure
to become consistent by renaming the conflicting objects via adding a
special suffix to them (e.g "_orig_2015-01-02"). Conflicting objects are
objects that already exist in Pithos as file objects but they should be
directory objects instead.

 b) --fix-dir-names will rename all existing directory objects that
contain backslashes in their names, replacing '\' with '/'.

 c) --fix-missing-dirs will create missing intermediate directory

The --yes option will bypass prompting the end user.
Signed-off-by: default avatarDimitris Aragiorgis <>
parent eb5f6929
......@@ -33,7 +33,123 @@
from datetime import date
from kamaki.cli import command
from kamaki.cli.errors import CLIError
from kamaki.cli.cmdtree import CommandTree
from kamaki.cli.cmds import errors, OptionalOutput
from kamaki.cli.cmds.pithos import _PithosAccount
from kamaki.cli.argument import FlagArgument
scripts_cmds = CommandTree('scripts', 'Useful scripts')
namespaces = [scripts_cmds, ]
class scripts_verifyfs(_PithosAccount, OptionalOutput):
"""Verify/Fix the structure of directory objects inside a container"""
arguments = dict(
'Fix conflicting names by renaming them '
'(prepare the structure of directory objects to be consistent)',
'Rename directory objects containing \\',
'Create missing directories objects',
yes=FlagArgument('Do not prompt for permission', '--yes'),
def _run(self):
dirs, files, empty_files = [], [], []
result = self.client.container_get()
for o in result.json:
name = o['name']
if self.object_is_dir(o):
elif o['bytes'] == 0:
# Find all directories with backslashes
wrong = set()
for d in dirs:
if '\\' in d:
# Find all intermediate directories and see if a missing directory
# exists or if an intermediate directory conflicts with an existing
# object name
missing = set()
conflicts = set()
for n in files + dirs:
inter = n.split('/')
d = []
for i in inter:
p = '/'.join(d)
if p not in dirs:
if p in files + empty_files:
# First try to resolve conflicts
if self['fix_conflicts']:
for c in conflicts:
if self['yes'] or self.ask_user('Rename %s?' % c):
backup = '%s_orig_%s' % (c,
# TODO: check if backup name already exists
self.error(' * Renaming %s to %s' % (c, backup))
elif conflicts:
raise CLIError(
'Conflicting object names found: %s' % conflicts,
'They should be directory objects instead',
'Use --fix-conflicts to rename them and prepare'
' the directory structure for further fix actions'])
# renames should take place after fixing conflicts
elif self['fix_names']:
for w in wrong:
correct = w.replace('\\', '/')
if self['yes'] or self.ask_user('Rename %s?' % w):
self.error(' * Renaming %s to %s' % (w, correct))
elif wrong:
raise CLIError(
'Directory objects with backslashes found: %s' % wrong,
details=['Use --fix-dir-names to sanitize them'])
# missing dirs should be created after fixing names
elif self['fix_missing']:
for d in missing:
if self['yes'] or self.ask_user('Create %s?' % d):
self.error(' * Creating directory object %s' % d)
elif missing:
raise CLIError(
'Missing directory objects found: %s' % missing,
details=['Use --fix-missing-dirs to create them'])
def main(self, container):
super(self.__class__, self)._run()
self.container, self.client.container = container, container
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment