utils.py 33.6 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#
Iustin Pop's avatar
Iustin Pop committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Ganeti small utilities
23

Iustin Pop's avatar
Iustin Pop committed
24
25
26
27
28
29
30
"""


import sys
import os
import sha
import time
31
import subprocess
Iustin Pop's avatar
Iustin Pop committed
32
33
34
35
import re
import socket
import tempfile
import shutil
36
import errno
37
import pwd
Guido Trotter's avatar
Guido Trotter committed
38
import itertools
39
40
import select
import fcntl
41
import resource
42
import logging
Michael Hanselmann's avatar
Michael Hanselmann committed
43
import signal
44
45

from cStringIO import StringIO
Iustin Pop's avatar
Iustin Pop committed
46
47

from ganeti import errors
Iustin Pop's avatar
Iustin Pop committed
48
from ganeti import constants
Iustin Pop's avatar
Iustin Pop committed
49

50

Iustin Pop's avatar
Iustin Pop committed
51
52
53
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')

54
debug = False
55
no_fork = False
56

57

Iustin Pop's avatar
Iustin Pop committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
class RunResult(object):
  """Simple class for holding the result of running external programs.

  Instance variables:
    exit_code: the exit code of the program, or None (if the program
               didn't exit())
    signal: numeric signal that caused the program to finish, or None
            (if the program wasn't terminated by a signal)
    stdout: the standard output of the program
    stderr: the standard error of the program
    failed: a Boolean value which is True in case the program was
            terminated by a signal or exited with a non-zero exit code
    fail_reason: a string detailing the termination reason

  """
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
               "failed", "fail_reason", "cmd"]


Iustin Pop's avatar
Iustin Pop committed
77
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
Iustin Pop's avatar
Iustin Pop committed
78
79
    self.cmd = cmd
    self.exit_code = exit_code
Iustin Pop's avatar
Iustin Pop committed
80
    self.signal = signal_
Iustin Pop's avatar
Iustin Pop committed
81
82
    self.stdout = stdout
    self.stderr = stderr
Iustin Pop's avatar
Iustin Pop committed
83
    self.failed = (signal_ is not None or exit_code != 0)
Iustin Pop's avatar
Iustin Pop committed
84
85
86
87
88
89
90
91

    if self.signal is not None:
      self.fail_reason = "terminated by signal %s" % self.signal
    elif self.exit_code is not None:
      self.fail_reason = "exited with exit code %s" % self.exit_code
    else:
      self.fail_reason = "unable to determine termination reason"

92
93
94
    if self.failed:
      logging.debug("Command '%s' failed (%s); output: %s",
                    self.cmd, self.fail_reason, self.output)
95

Iustin Pop's avatar
Iustin Pop committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
  def _GetOutput(self):
    """Returns the combined stdout and stderr for easier usage.

    """
    return self.stdout + self.stderr

  output = property(_GetOutput, None, None, "Return full output")


def RunCmd(cmd):
  """Execute a (shell) command.

  The command should not read from its standard input, as it will be
  closed.

  Args:
    cmd: command to run. (str)

  Returns: `RunResult` instance

  """
117
118
119
  if no_fork:
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")

Iustin Pop's avatar
Iustin Pop committed
120
121
  if isinstance(cmd, list):
    cmd = [str(val) for val in cmd]
122
123
124
125
126
    strcmd = " ".join(cmd)
    shell = False
  else:
    strcmd = cmd
    shell = True
127
  logging.debug("RunCmd '%s'", strcmd)
128
129
  env = os.environ.copy()
  env["LC_ALL"] = "C"
130
  poller = select.poll()
131
132
133
134
  child = subprocess.Popen(cmd, shell=shell,
                           stderr=subprocess.PIPE,
                           stdout=subprocess.PIPE,
                           stdin=subprocess.PIPE,
135
                           close_fds=True, env=env)
136
137

  child.stdin.close()
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  poller.register(child.stdout, select.POLLIN)
  poller.register(child.stderr, select.POLLIN)
  out = StringIO()
  err = StringIO()
  fdmap = {
    child.stdout.fileno(): (out, child.stdout),
    child.stderr.fileno(): (err, child.stderr),
    }
  for fd in fdmap:
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)

  while fdmap:
    for fd, event in poller.poll():
      if event & select.POLLIN or event & select.POLLPRI:
        data = fdmap[fd][1].read()
        # no data from read signifies EOF (the same as POLLHUP)
        if not data:
          poller.unregister(fd)
          del fdmap[fd]
          continue
        fdmap[fd][0].write(data)
      if (event & select.POLLNVAL or event & select.POLLHUP or
          event & select.POLLERR):
        poller.unregister(fd)
        del fdmap[fd]

  out = out.getvalue()
  err = err.getvalue()
Iustin Pop's avatar
Iustin Pop committed
167
168

  status = child.wait()
169
170
  if status >= 0:
    exitcode = status
Iustin Pop's avatar
Iustin Pop committed
171
    signal_ = None
Iustin Pop's avatar
Iustin Pop committed
172
173
  else:
    exitcode = None
Iustin Pop's avatar
Iustin Pop committed
174
    signal_ = -status
Iustin Pop's avatar
Iustin Pop committed
175

Iustin Pop's avatar
Iustin Pop committed
176
  return RunResult(exitcode, signal_, out, err, strcmd)
Iustin Pop's avatar
Iustin Pop committed
177
178
179
180
181
182
183
184
185
186
187
188


def RemoveFile(filename):
  """Remove a file ignoring some errors.

  Remove a file, ignoring non-existing ones or directories. Other
  errors are passed.

  """
  try:
    os.unlink(filename)
  except OSError, err:
189
    if err.errno not in (errno.ENOENT, errno.EISDIR):
Iustin Pop's avatar
Iustin Pop committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
      raise


def _FingerprintFile(filename):
  """Compute the fingerprint of a file.

  If the file does not exist, a None will be returned
  instead.

  Args:
    filename - Filename (str)

  """
  if not (os.path.exists(filename) and os.path.isfile(filename)):
    return None

  f = open(filename)

  fp = sha.sha()
  while True:
    data = f.read(4096)
    if not data:
      break

    fp.update(data)

  return fp.hexdigest()


def FingerprintFiles(files):
  """Compute fingerprints for a list of files.

  Args:
    files - array of filenames.  ( [str, ...] )

  Return value:
    dictionary of filename: fingerprint for the files that exist

  """
  ret = {}

  for filename in files:
    cksum = _FingerprintFile(filename)
    if cksum:
      ret[filename] = cksum

  return ret


def CheckDict(target, template, logname=None):
  """Ensure a dictionary has a required set of keys.

  For the given dictionaries `target` and `template`, ensure target
  has all the keys from template. Missing keys are added with values
  from template.

  Args:
    target   - the dictionary to check
    template - template dictionary
    logname  - a caller-chosen string to identify the debug log
               entry; if None, no logging will be done

  Returns value:
    None

  """
  missing = []
  for k in template:
    if k not in target:
      missing.append(k)
      target[k] = template[k]

  if missing and logname:
263
    logging.warning('%s missing keys %s', logname, ', '.join(missing))
Iustin Pop's avatar
Iustin Pop committed
264
265
266
267
268
269
270


def IsProcessAlive(pid):
  """Check if a given pid exists on the system.

  Returns: true or false, depending on if the pid exists or not

271
272
  Remarks: zombie processes treated as not alive, and giving a pid <=
  0 makes the function to return False.
Iustin Pop's avatar
Iustin Pop committed
273
274

  """
275
276
277
  if pid <= 0:
    return False

Iustin Pop's avatar
Iustin Pop committed
278
279
280
  try:
    f = open("/proc/%d/status" % pid)
  except IOError, err:
281
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
Iustin Pop's avatar
Iustin Pop committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
      return False

  alive = True
  try:
    data = f.readlines()
    if len(data) > 1:
      state = data[1].split()
      if len(state) > 1 and state[1] == "Z":
        alive = False
  finally:
    f.close()

  return alive


297
298
def ReadPidFile(pidfile):
  """Read the pid from a file.
299

300
301
302
303
304
  @param pidfile: Path to a file containing the pid to be checked
  @type  pidfile: string (filename)
  @return: The process id, if the file exista and contains a valid PID,
           otherwise 0
  @rtype: int
305
306
307
308

  """
  try:
    pf = open(pidfile, 'r')
309
310
311
312
  except EnvironmentError, err:
    if err.errno != errno.ENOENT:
      logging.exception("Can't read pid file?!")
    return 0
313
314
315

  try:
    pid = int(pf.read())
316
  except ValueError, err:
317
    logging.info("Can't parse pid file contents", exc_info=True)
318
    return 0
319

320
  return pid
321
322


Iustin Pop's avatar
Iustin Pop committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
def MatchNameComponent(key, name_list):
  """Try to match a name against a list.

  This function will try to match a name like test1 against a list
  like ['test1.example.com', 'test2.example.com', ...]. Against this
  list, 'test1' as well as 'test1.example' will match, but not
  'test1.ex'. A multiple match will be considered as no match at all
  (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).

  Args:
    key: the name to be searched
    name_list: the list of strings against which to search the key

  Returns:
    None if there is no match *or* if there are multiple matches
    otherwise the element from the list which matches

  """
  mo = re.compile("^%s(\..*)?$" % re.escape(key))
  names_filtered = [name for name in name_list if mo.match(name) is not None]
  if len(names_filtered) != 1:
    return None
  return names_filtered[0]


348
class HostInfo:
349
  """Class implementing resolver and hostname functionality
350
351

  """
352
  def __init__(self, name=None):
353
354
    """Initialize the host name object.

355
356
    If the name argument is not passed, it will use this system's
    name.
357
358

    """
359
360
361
362
363
    if name is None:
      name = self.SysName()

    self.query = name
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
364
365
    self.ip = self.ipaddrs[0]

366
367
368
369
370
371
  def ShortName(self):
    """Returns the hostname without domain.

    """
    return self.name.split('.')[0]

372
373
374
  @staticmethod
  def SysName():
    """Return the current system's name.
375

376
    This is simply a wrapper over socket.gethostname()
Iustin Pop's avatar
Iustin Pop committed
377

378
379
    """
    return socket.gethostname()
Iustin Pop's avatar
Iustin Pop committed
380

381
382
383
  @staticmethod
  def LookupHostname(hostname):
    """Look up hostname
Iustin Pop's avatar
Iustin Pop committed
384

385
386
387
388
389
390
391
392
393
394
395
396
397
    Args:
      hostname: hostname to look up

    Returns:
      a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
      in case of errors in resolving, we raise a ResolverError

    """
    try:
      result = socket.gethostbyname_ex(hostname)
    except socket.gaierror, err:
      # hostname not found in DNS
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
Iustin Pop's avatar
Iustin Pop committed
398

399
    return result
Iustin Pop's avatar
Iustin Pop committed
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419


def ListVolumeGroups():
  """List volume groups and their size

  Returns:
     Dictionary with keys volume name and values the size of the volume

  """
  command = "vgs --noheadings --units m --nosuffix -o name,size"
  result = RunCmd(command)
  retval = {}
  if result.failed:
    return retval

  for line in result.stdout.splitlines():
    try:
      name, size = line.split()
      size = int(float(size))
    except (IndexError, ValueError), err:
420
      logging.error("Invalid output from vgs (%s): %s", err, line)
Iustin Pop's avatar
Iustin Pop committed
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
      continue

    retval[name] = size

  return retval


def BridgeExists(bridge):
  """Check whether the given bridge exists in the system

  Returns:
     True if it does, false otherwise.

  """
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)


def NiceSort(name_list):
  """Sort a list of strings based on digit and non-digit groupings.

  Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
  sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].

  The sort algorithm breaks each name in groups of either only-digits
  or no-digits. Only the first eight such groups are considered, and
  after that we just use what's left of the string.

  Return value
    - a copy of the list sorted according to our algorithm

  """
  _SORTER_BASE = "(\D+|\d+)"
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE)
  _SORTER_RE = re.compile(_SORTER_FULL)
  _SORTER_NODIGIT = re.compile("^\D*$")
  def _TryInt(val):
    """Attempts to convert a variable to integer."""
    if val is None or _SORTER_NODIGIT.match(val):
      return val
    rval = int(val)
    return rval

  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
             for name in name_list]
  to_sort.sort()
  return [tup[1] for tup in to_sort]


def TryConvert(fn, val):
  """Try to convert a value ignoring errors.

  This function tries to apply function `fn` to `val`. If no
  ValueError or TypeError exceptions are raised, it will return the
  result, else it will return the original value. Any other exceptions
  are propagated to the caller.

  """
  try:
    nv = fn(val)
  except (ValueError, TypeError), err:
    nv = val
  return nv


def IsValidIP(ip):
  """Verifies the syntax of an IP address.

  This function checks if the ip address passes is valid or not based
  on syntax (not ip range, class calculations or anything).

  """
  unit = "(0|[1-9]\d{0,2})"
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)


def IsValidShellParam(word):
  """Verifies is the given word is safe from the shell's p.o.v.

  This means that we can pass this to a command via the shell and be
  sure that it doesn't alter the command line and is passed as such to
  the actual command.

  Note that we are overly restrictive here, in order to be on the safe
  side.

  """
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))


def BuildShellCmd(template, *args):
  """Build a safe shell command line from the given arguments.

  This function will check all arguments in the args list so that they
  are valid shell parameters (i.e. they don't contain shell
  metacharaters). If everything is ok, it will return the result of
  template % args.

  """
  for word in args:
    if not IsValidShellParam(word):
524
525
      raise errors.ProgrammerError("Shell argument '%s' contains"
                                   " invalid characters" % word)
Iustin Pop's avatar
Iustin Pop committed
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
  return template % args


def FormatUnit(value):
  """Formats an incoming number of MiB with the appropriate unit.

  Value needs to be passed as a numeric type. Return value is always a string.

  """
  if value < 1024:
    return "%dM" % round(value, 0)

  elif value < (1024 * 1024):
    return "%0.1fG" % round(float(value) / 1024, 1)

  else:
    return "%0.1fT" % round(float(value) / 1024 / 1024, 1)


def ParseUnit(input_string):
  """Tries to extract number and scale from the given string.

  Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
  is specified, it defaults to MiB. Return value is always an int in MiB.

  """
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
  if not m:
554
    raise errors.UnitParseError("Invalid format")
Iustin Pop's avatar
Iustin Pop committed
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574

  value = float(m.groups()[0])

  unit = m.groups()[1]
  if unit:
    lcunit = unit.lower()
  else:
    lcunit = 'm'

  if lcunit in ('m', 'mb', 'mib'):
    # Value already in MiB
    pass

  elif lcunit in ('g', 'gb', 'gib'):
    value *= 1024

  elif lcunit in ('t', 'tb', 'tib'):
    value *= 1024 * 1024

  else:
575
    raise errors.UnitParseError("Unknown unit: %s" % unit)
Iustin Pop's avatar
Iustin Pop committed
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626

  # Make sure we round up
  if int(value) < value:
    value += 1

  # Round up to the next multiple of 4
  value = int(value)
  if value % 4:
    value += 4 - value % 4

  return value


def AddAuthorizedKey(file_name, key):
  """Adds an SSH public key to an authorized_keys file.

  Args:
    file_name: Path to authorized_keys file
    key: String containing key
  """
  key_fields = key.split()

  f = open(file_name, 'a+')
  try:
    nl = True
    for line in f:
      # Ignore whitespace changes
      if line.split() == key_fields:
        break
      nl = line.endswith('\n')
    else:
      if not nl:
        f.write("\n")
      f.write(key.rstrip('\r\n'))
      f.write("\n")
      f.flush()
  finally:
    f.close()


def RemoveAuthorizedKey(file_name, key):
  """Removes an SSH public key from an authorized_keys file.

  Args:
    file_name: Path to authorized_keys file
    key: String containing key
  """
  key_fields = key.split()

  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
627
    out = os.fdopen(fd, 'w')
Iustin Pop's avatar
Iustin Pop committed
628
    try:
629
630
631
632
633
634
      f = open(file_name, 'r')
      try:
        for line in f:
          # Ignore whitespace changes while comparing lines
          if line.split() != key_fields:
            out.write(line)
635
636
637
638
639
640
641
642
643
644
645
646

        out.flush()
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise


647
648
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
  """Sets the name of an IP address and hostname in /etc/hosts.
649
650

  """
651
652
653
  # Ensure aliases are unique
  aliases = UniqueSequence([hostname] + aliases)[1:]

654
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
655
  try:
656
657
658
659
660
661
662
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        written = False
        for line in f:
          fields = line.split()
663
          if fields and not fields[0].startswith('#') and ip == fields[0]:
664
665
666
            continue
          out.write(line)

667
        out.write("%s\t%s" % (ip, hostname))
668
669
670
671
672
        if aliases:
          out.write(" %s" % ' '.join(aliases))
        out.write('\n')

        out.flush()
673
        os.fsync(out)
674
675
676
677
678
679
680
681
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise
682
683


684
685
686
687
688
689
690
691
def AddHostToEtcHosts(hostname):
  """Wrapper around SetEtcHostsEntry.

  """
  hi = HostInfo(name=hostname)
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])


692
def RemoveEtcHostsEntry(file_name, hostname):
693
  """Removes a hostname from /etc/hosts.
694

695
  IP addresses without names are removed from the file.
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
  """
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        for line in f:
          fields = line.split()
          if len(fields) > 1 and not fields[0].startswith('#'):
            names = fields[1:]
            if hostname in names:
              while hostname in names:
                names.remove(hostname)
              if names:
711
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
712
713
714
              continue

          out.write(line)
715
716

        out.flush()
717
        os.fsync(out)
718
719
720
        os.rename(tmpname, file_name)
      finally:
        f.close()
Iustin Pop's avatar
Iustin Pop committed
721
    finally:
722
723
724
725
      out.close()
  except:
    RemoveFile(tmpname)
    raise
Iustin Pop's avatar
Iustin Pop committed
726
727


728
729
730
731
732
733
734
735
736
def RemoveHostFromEtcHosts(hostname):
  """Wrapper around RemoveEtcHostsEntry.

  """
  hi = HostInfo(name=hostname)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())


Iustin Pop's avatar
Iustin Pop committed
737
738
739
740
741
742
743
def CreateBackup(file_name):
  """Creates a backup of a file.

  Returns: the path to the newly created backup file.

  """
  if not os.path.isfile(file_name):
744
745
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
                                file_name)
Iustin Pop's avatar
Iustin Pop committed
746

747
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
Iustin Pop's avatar
Iustin Pop committed
748
  dir_name = os.path.dirname(file_name)
749
750
751

  fsrc = open(file_name, 'rb')
  try:
Iustin Pop's avatar
Iustin Pop committed
752
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
753
754
755
756
757
758
759
760
    fdst = os.fdopen(fd, 'wb')
    try:
      shutil.copyfileobj(fsrc, fdst)
    finally:
      fdst.close()
  finally:
    fsrc.close()

Iustin Pop's avatar
Iustin Pop committed
761
762
763
764
765
  return backup_name


def ShellQuote(value):
  """Quotes shell argument according to POSIX.
766

Iustin Pop's avatar
Iustin Pop committed
767
768
769
770
771
772
773
774
775
776
777
778
  """
  if _re_shell_unquoted.match(value):
    return value
  else:
    return "'%s'" % value.replace("'", "'\\''")


def ShellQuoteArgs(args):
  """Quotes all given shell arguments and concatenates using spaces.

  """
  return ' '.join([ShellQuote(i) for i in args])
779
780


781
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
782
783
  """Simple ping implementation using TCP connect(2).

784
785
786
787
788
789
790
791
  Try to do a TCP connect(2) from an optional source IP to the
  specified target IP and the specified target port. If the optional
  parameter live_port_needed is set to true, requires the remote end
  to accept the connection. The timeout is specified in seconds and
  defaults to 10 seconds. If the source optional argument is not
  passed, the source address selection is left to the kernel,
  otherwise we try to connect using the passed address (failures to
  bind other than EADDRNOTAVAIL will be ignored).
792
793
794
795
796
797

  """
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

  sucess = False

798
799
800
801
802
803
  if source is not None:
    try:
      sock.bind((source, 0))
    except socket.error, (errcode, errstring):
      if errcode == errno.EADDRNOTAVAIL:
        success = False
804
805
806
807
808
809
810
811
812
813

  sock.settimeout(timeout)

  try:
    sock.connect((target, port))
    sock.close()
    success = True
  except socket.timeout:
    success = False
  except socket.error, (errcode, errstring):
814
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
815
816

  return success
817
818
819
820
821
822


def ListVisibleFiles(path):
  """Returns a list of all visible files in a directory.

  """
823
824
825
  files = [i for i in os.listdir(path) if not i.startswith(".")]
  files.sort()
  return files
826
827


828
829
830
831
832
833
def GetHomeDir(user, default=None):
  """Try to get the homedir of the given user.

  The user can be passed either as a string (denoting the name) or as
  an integer (denoting the user id). If the user is not found, the
  'default' argument is returned, which defaults to None.
834
835
836

  """
  try:
837
838
839
840
841
842
843
    if isinstance(user, basestring):
      result = pwd.getpwnam(user)
    elif isinstance(user, (int, long)):
      result = pwd.getpwuid(user)
    else:
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
                                   type(user))
844
845
846
  except KeyError:
    return default
  return result.pw_dir
847
848


849
def NewUUID():
850
851
852
853
854
855
856
857
  """Returns a random UUID.

  """
  f = open("/proc/sys/kernel/random/uuid", "r")
  try:
    return f.read(128).rstrip("\n")
  finally:
    f.close()
Iustin Pop's avatar
Iustin Pop committed
858
859
860
861


def WriteFile(file_name, fn=None, data=None,
              mode=None, uid=-1, gid=-1,
862
              atime=None, mtime=None, close=True,
863
              dry_run=False, backup=False,
864
              prewrite=None, postwrite=None):
Iustin Pop's avatar
Iustin Pop committed
865
866
867
868
869
870
871
872
873
874
875
876
877
  """(Over)write a file atomically.

  The file_name and either fn (a function taking one argument, the
  file descriptor, and which should write the data to it) or data (the
  contents of the file) must be passed. The other arguments are
  optional and allow setting the file mode, owner and group, and the
  mtime/atime of the file.

  If the function doesn't raise an exception, it has succeeded and the
  target file has the new contents. If the file has raised an
  exception, an existing target file should be unmodified and the
  temporary file should be removed.

878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
  Args:
    file_name: New filename
    fn: Content writing function, called with file descriptor as parameter
    data: Content as string
    mode: File mode
    uid: Owner
    gid: Group
    atime: Access time
    mtime: Modification time
    close: Whether to close file after writing it
    prewrite: Function object called before writing content
    postwrite: Function object called after writing content

  Returns:
    None if "close" parameter evaluates to True, otherwise file descriptor.

Iustin Pop's avatar
Iustin Pop committed
894
  """
895
  if not os.path.isabs(file_name):
Iustin Pop's avatar
Iustin Pop committed
896
897
898
899
900
901
902
903
904
905
    raise errors.ProgrammerError("Path passed to WriteFile is not"
                                 " absolute: '%s'" % file_name)

  if [fn, data].count(None) != 1:
    raise errors.ProgrammerError("fn or data required")

  if [atime, mtime].count(None) == 1:
    raise errors.ProgrammerError("Both atime and mtime must be either"
                                 " set or None")

Michael Hanselmann's avatar
Michael Hanselmann committed
906
907
  if backup and not dry_run and os.path.isfile(file_name):
    CreateBackup(file_name)
Iustin Pop's avatar
Iustin Pop committed
908
909
910
911
912
913
914
915
916
917

  dir_name, base_name = os.path.split(file_name)
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
  # here we need to make sure we remove the temp file, if any error
  # leaves it in place
  try:
    if uid != -1 or gid != -1:
      os.chown(new_name, uid, gid)
    if mode:
      os.chmod(new_name, mode)
918
919
    if callable(prewrite):
      prewrite(fd)
Iustin Pop's avatar
Iustin Pop committed
920
921
922
923
    if data is not None:
      os.write(fd, data)
    else:
      fn(fd)
924
925
    if callable(postwrite):
      postwrite(fd)
Iustin Pop's avatar
Iustin Pop committed
926
927
928
    os.fsync(fd)
    if atime is not None and mtime is not None:
      os.utime(new_name, (atime, mtime))
Michael Hanselmann's avatar
Michael Hanselmann committed
929
930
    if not dry_run:
      os.rename(new_name, file_name)
Iustin Pop's avatar
Iustin Pop committed
931
  finally:
932
933
934
935
936
    if close:
      os.close(fd)
      result = None
    else:
      result = fd
Iustin Pop's avatar
Iustin Pop committed
937
    RemoveFile(new_name)
Guido Trotter's avatar
Guido Trotter committed
938

939
940
  return result

Guido Trotter's avatar
Guido Trotter committed
941

942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
def FirstFree(seq, base=0):
  """Returns the first non-existing integer from seq.

  The seq argument should be a sorted list of positive integers. The
  first time the index of an element is smaller than the element
  value, the index will be returned.

  The base argument is used to start at a different offset,
  i.e. [3, 4, 6] with offset=3 will return 5.

  Example: [0, 1, 3] will return 2.

  """
  for idx, elem in enumerate(seq):
    assert elem >= base, "Passed element is higher than base offset"
    if elem > idx + base:
      # idx is not used
      return idx + base
  return None


Guido Trotter's avatar
Guido Trotter committed
963
964
965
966
967
968
969
970
971
972
973
974
def all(seq, pred=bool):
  "Returns True if pred(x) is True for every element in the iterable"
  for elem in itertools.ifilterfalse(pred, seq):
    return False
  return True


def any(seq, pred=bool):
  "Returns True if pred(x) is True for at least one element in the iterable"
  for elem in itertools.ifilter(pred, seq):
    return True
  return False
975
976
977
978
979
980
981
982
983


def UniqueSequence(seq):
  """Returns a list with unique elements.

  Element order is preserved.
  """
  seen = set()
  return [i for i in seq if i not in seen and not seen.add(i)]
984
985
986
987
988
989
990
991
992
993


def IsValidMac(mac):
  """Predicate to check if a MAC address is valid.

  Checks wether the supplied MAC address is formally correct, only
  accepts colon separated format.
  """
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
  return mac_check.match(mac) is not None
994
995
996
997
998
999
1000
1001
1002
1003


def TestDelay(duration):
  """Sleep for a fixed amount of time.

  """
  if duration < 0:
    return False
  time.sleep(duration)
  return True
1004
1005


1006
def Daemonize(logfile, noclose_fds=None):
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
  """Daemonize the current process.

  This detaches the current process from the controlling terminal and
  runs it in the background as a daemon.

  """
  UMASK = 077
  WORKDIR = "/"
  # Default maximum for the number of available file descriptors.
  if 'SC_OPEN_MAX' in os.sysconf_names:
    try:
      MAXFD = os.sysconf('SC_OPEN_MAX')
      if MAXFD < 0:
        MAXFD = 1024
    except OSError:
      MAXFD = 1024
  else:
    MAXFD = 1024

  # this might fail
  pid = os.fork()
  if (pid == 0):  # The first child.
    os.setsid()
    # this might fail
    pid = os.fork() # Fork a second child.
    if (pid == 0):  # The second child.
      os.chdir(WORKDIR)
      os.umask(UMASK)
    else:
      # exit() or _exit()?  See below.
      os._exit(0) # Exit parent (the first child) of the second child.
  else:
    os._exit(0) # Exit parent of the first child.
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
  if (maxfd == resource.RLIM_INFINITY):
    maxfd = MAXFD

  # Iterate through and close all file descriptors.
  for fd in range(0, maxfd):
1046
1047
    if noclose_fds and fd in noclose_fds:
      continue
1048
1049
1050
1051
1052
1053
1054
1055
1056
    try:
      os.close(fd)
    except OSError: # ERROR, fd wasn't open to begin with (ignored)
      pass
  os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
  # Duplicate standard input to standard output and standard error.
  os.dup2(0, 1)     # standard output (1)
  os.dup2(0, 2)     # standard error (2)
  return 0
1057
1058


Iustin Pop's avatar
Iustin Pop committed
1059
def DaemonPidFileName(name):
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
  """Compute a ganeti pid file absolute path, given the daemon name.

  """
  return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)


def WritePidFile(name):
  """Write the current process pidfile.

  The file will be written to constants.RUN_GANETI_DIR/name.pid

  """
  pid = os.getpid()
Iustin Pop's avatar
Iustin Pop committed
1073
  pidfilename = DaemonPidFileName(name)
1074
  if IsProcessAlive(ReadPidFile(pidfilename)):
1075
    raise errors.GenericError("%s contains a live process" % pidfilename)
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086

  WriteFile(pidfilename, data="%d\n" % pid)


def RemovePidFile(name):
  """Remove the current process pidfile.

  Any errors are ignored.

  """
  pid = os.getpid()
Iustin Pop's avatar
Iustin Pop committed
1087
  pidfilename = DaemonPidFileName(name)
1088
1089
1090
1091
1092
1093
1094
  # TODO: we could check here that the file contains our pid
  try:
    RemoveFile(pidfilename)
  except:
    pass


Iustin Pop's avatar
Iustin Pop committed
1095
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30):
Iustin Pop's avatar
Iustin Pop committed
1096
1097
1098
1099
  """Kill a process given by its pid.

  @type pid: int
  @param pid: The PID to terminate.
Iustin Pop's avatar
Iustin Pop committed
1100
1101
  @type signal_: int
  @param signal_: The signal to send, by default SIGTERM
Iustin Pop's avatar
Iustin Pop committed
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
  @type timeout: int
  @param timeout: The timeout after which, if the process is still alive,
                  a SIGKILL will be sent. If not positive, no such checking
                  will be done

  """
  if pid <= 0:
    # kill with pid=0 == suicide
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)

  if not IsProcessAlive(pid):
    return
Iustin Pop's avatar
Iustin Pop committed
1114
  os.kill(pid, signal_)
Iustin Pop's avatar
Iustin Pop committed
1115
1116
1117
1118
1119
1120
1121
1122
1123
  if timeout <= 0:
    return
  end = time.time() + timeout
  while time.time() < end and IsProcessAlive(pid):
    time.sleep(0.1)
  if IsProcessAlive(pid):
    os.kill(pid, signal.SIGKILL)


1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
def FindFile(name, search_path, test=os.path.exists):
  """Look for a filesystem object in a given path.

  This is an abstract method to search for filesystem object (files,
  dirs) under a given search path.

  Args:
    - name: the name to look for
    - search_path: list of directory names
    - test: the test which the full path must satisfy
      (defaults to os.path.exists)

  Returns:
    - full path to the item if found
    - None otherwise

  """
  for dir_name in search_path:
    item_name = os.path.sep.join([dir_name, name])
    if test(item_name):
      return item_name
  return None
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161


def CheckVolumeGroupSize(vglist, vgname, minsize):
  """Checks if the volume group list is valid.

  A non-None return value means there's an error, and the return value
  is the error message.

  """
  vgsize = vglist.get(vgname, None)
  if vgsize is None:
    return "volume group '%s' missing" % vgname
  elif vgsize < minsize:
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
            (vgname, minsize, vgsize))
  return None
1162
1163


1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
def SplitTime(seconds):
  """Splits time as floating point number into a tuple.

  @param seconds: Time in seconds
  @type seconds: int or float
  @return: Tuple containing (seconds, milliseconds)

  """
  (seconds, fraction) = divmod(seconds, 1.0)
  return (int(seconds), int(round(fraction * 1000, 0)))


def MergeTime(timetuple):
  """Merges a tuple into time as a floating point number.

  @param timetuple: Time as tuple, (seconds, milliseconds)
  @type timetuple: tuple
  @return: Time as a floating point number expressed in seconds

  """
  (seconds, milliseconds) = timetuple

  assert 0 <= seconds, "Seconds must be larger than 0"
  assert 0 <= milliseconds <= 999, "Milliseconds must be 0-999"

  return float(seconds) + (float(1) / 1000 * milliseconds)


1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
def LockedMethod(fn):
  """Synchronized object access decorator.

  This decorator is intended to protect access to an object using the
  object's own lock which is hardcoded to '_lock'.

  """
  def wrapper(self, *args, **kwargs):
    assert hasattr(self, '_lock')
    lock = self._lock
    lock.acquire()
    try:
      result = fn(self, *args, **kwargs)
    finally:
      lock.release()
    return result
  return wrapper
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220


def LockFile(fd):
  """Locks a file using POSIX locks.

  """
  try:
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  except IOError, err:
    if err.errno == errno.EAGAIN:
      raise errors.LockError("File already locked")
    raise
Michael Hanselmann's avatar
Michael Hanselmann committed
1221
1222


1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
class FileLock(object):
  """Utility class for file locks.

  """
  def __init__(self, filename):
    self.filename = filename
    self.fd = open(self.filename, "w")

  def __del__(self):
    self.Close()

  def Close(self):
    if self.fd:
      self.fd.close()
      self.fd = None

  def _flock(self, flag, blocking, errmsg):
    assert self.fd, "Lock was closed"

    if not blocking:
      flag |= fcntl.LOCK_NB

    try:
      fcntl.flock(self.fd, flag)
    except IOError, err:
      if err.errno in (errno.EAGAIN, ):
        raise errors.LockError(errmsg)
1250
1251
1252
      else:
        logging.exception("fcntl.flock failed")
        raise
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279

  def Exclusive(self, blocking=False):
    """Locks the file in exclusive mode.

    """
    self._flock(fcntl.LOCK_EX, blocking,
                "Failed to lock %s in exclusive mode" % self.filename)

  def Shared(self, blocking=False):
    """Locks the file in shared mode.

    """
    self._flock(fcntl.LOCK_SH, blocking,
                "Failed to lock %s in shared mode" % self.filename)

  def Unlock(self, blocking=True):
    """Unlocks the file.

    According to "man flock", unlocking can also be a nonblocking operation:
    "To make a non-blocking request, include LOCK_NB with any of the above
    operations"

    """
    self._flock(fcntl.LOCK_UN, blocking,
                "Failed to unlock %s" % self.filename)


Michael Hanselmann's avatar
Michael Hanselmann committed
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
class SignalHandler(object):
  """Generic signal handler class.

  It automatically restores the original handler when deconstructed or when
  Reset() is called. You can either pass your own handler function in or query
  the "called" attribute to detect whether the signal was sent.

  """
  def __init__(self, signum):
    """Constructs a new SignalHandler instance.

    @param signum: Single signal number or set of signal numbers

    """
    if isinstance(signum, (int, long)):
      self.signum = set([signum])
    else:
      self.signum = set(signum)

    self.called = False

    self._previous = {}
    try:
      for signum in self.signum:
        # Setup handler
        prev_handler = signal.signal(signum, self._HandleSignal)
        try:
          self._previous[signum] = prev_handler
        except:
          # Restore previous handler
          signal.signal(signum, prev_handler)
          raise
    except:
      # Reset all handlers
      self.Reset()
      # Here we have a race condition: a handler may have already been called,
      # but there's not much we can do about it at this point.
      raise

  def __del__(self):
    self.Reset()

  def Reset(self):
    """Restore previous handler.

    """
    for signum, prev_handler in self._previous.items():
      signal.signal(signum, prev_handler)
      # If successful, remove from dict
      del self._previous[signum]

  def Clear(self):
    """Unsets "called" flag.

    This function can be used in case a signal may arrive several times.

    """
    self.called = False

  def _HandleSignal(self, signum, frame):
    """Actual signal handling function.

    """
    # This is not nice and not absolutely atomic, but it appears to be the only
    # solution in Python -- there are no atomic types.
    self.called = True