timeview.py 5.24 KB
Newer Older
Giorgos Korfiatis's avatar
Giorgos Korfiatis committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (C) 2015 GRNET S.A.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

16 17
from os import listdir, makedirs
from os.path import exists, isdir
Giorgos Korfiatis's avatar
Giorgos Korfiatis committed
18
import os
19 20 21 22 23 24 25 26
from subprocess import Popen, PIPE
import json
import errno


path_types = {'reg', 'dir', 'sym', 'fifo', 'sock', 'chr', 'blk'}


Efthymia Bika's avatar
Efthymia Bika committed
27 28 29 30 31 32
def path_join(base, *args):
    slash = base[0:1]
    if slash != '/':
        slash = ''
    base = slash + base.strip('/') + '/'
    return base + '/'.join(str(x).strip('/') for x in args)
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59


def timestamp_rfc822_to_iso8601(string):
    from email.utils import parsedate
    year, month, day, hour, minute, second, _, _, _ = parsedate(string)
    return "%04u-%02u-%02uT%02u:%02u:%02u" % (
        year, month, day, hour, minute, second)


def print_listing(openfile, root, path=()):
    openfile.write(root.get('timestamp', "- -")
                   + ' ' +  root.get('type', '-')
                   + ' ' + '/'.join(path) + '\0\n')
    for name, entry in root['entries'].iteritems():
        print_listing(openfile, entry, path=path + (name,))


def put_timepoint(config, timepoint, data):
    root = {'entries': {}}
    lines = data.split('\0')
    for i, line in enumerate(lines):
        if i & 1 == 0:
            continue
        line.strip('\n').strip('\0')
        if not line:
            continue

Efthymia Bika's avatar
Efthymia Bika committed
60
        parts = line.split(' ', 2)
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
        entry_type = parts[0]
        if entry_type not in path_types:
            continue

        timestamp = ' '.join(parts[1:-1])
        segments = parts[-1].split('/')
        node = root
        for segment in segments:
            entries = node['entries']
            if segment not in entries:
                entries[segment] = {'entries': {}}
            node = entries[segment]

        node['timestamp'] = timestamp
        node['type'] = entry_type

    datafile = path_join(config['datapath'], timepoint)
    with open(datafile, "w") as f:
        f.write(json.dumps(root, indent=2))


82
def fetch_timepoint(config, timepoint, cacert_file):
Giorgos Korfiatis's avatar
Giorgos Korfiatis committed
83 84 85
    curpath = os.path.dirname(os.path.realpath(__file__))
    duplicity = os.path.join(curpath, 'duplicity')
    args = [duplicity, 'list-current-files', '-t', timepoint,
86 87
            '--ssl-cacert-file', cacert_file, config['target_url']]

88 89 90 91 92 93 94 95 96
    proc = Popen(args, stdout=PIPE, stderr=PIPE)
    procout, procerr = proc.communicate()
    proc.wait()
    if proc.returncode != 0:
        raise RuntimeError(procerr)

    put_timepoint(config, timepoint, procout)


97
def get_timepoint(config, timepoint, path, cacert_file):
98 99 100 101 102 103 104 105 106 107 108
    datafile = path_join(config['datapath'], timepoint)
    retries = 1
    while True:
        try:
            with open(datafile) as f:
                data = f.read()
            break
        except IOError as e:
            if e.errno != errno.ENOENT or retries == 0:
                raise

109
        fetch_timepoint(config, timepoint, cacert_file)
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
        if retries <= 0:
            raise e
        retries -= 1

    node = json.loads(data)
    path = path.strip('/').split('/')
    for i in xrange(len(path)):
        segment = path[i]
        if not segment:
            continue

        entries = node['entries']
        if segment not in entries:
            return []

        node = entries[segment]

    return sorted({'name': name,
                   'type': entry['type'],
                   'timestamp': entry['timestamp']}
                  for name, entry in node['entries'].iteritems())


def ensure_datapath(datapath):
    if not exists(datapath):
        makedirs(datapath)
    elif not isdir(datapath):
        m = "%r not a directory" % datapath
        raise IOError(m)


def get_config():
    with open("timeview.config") as f:
        data = f.read()

    config = literal_eval(data)
    datapath = config['datapath']
    ensure_datapath(datapath)
    return config


def main():
    from sys import argv, stdin, stdout
    def help():
154
        print "Usage: %s <datapath> <target_url> <cacert_file> [get <absolute_timepoint> <path> | list]" % argv[0]
155 156
        raise SystemExit(1)

157
    if len(argv) < 5:
158 159 160 161 162 163
        help()

    datapath = argv[1]
    ensure_datapath(datapath)
    target_url = argv[2]

164
    cmd = argv[4]
165 166 167
    if cmd not in ['get', 'list']:
        help()

168
    cacert_file = argv[3]
169
    #config = get_config()
170 171 172
    config = {'datapath': datapath,
            'target_url': target_url,
            'cacert_file': cacert_file}
173 174

    if cmd == 'get':
175
        if len(argv) < 7:
176 177
            help()

178 179 180
        timepoint = argv[5]
        path = unicode(argv[6], encoding='UTF-8')
        r = get_timepoint(config, timepoint, path, cacert_file)
181 182 183 184 185 186 187 188 189 190 191
        print json.dumps(r, indent=2)

    elif cmd == 'list':
        print json.dumps(sorted(listdir(config['datapath'])), indent=2)

    else:
        help()


if __name__ == "__main__":
    main()