utils.py 36.7 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#
Iustin Pop's avatar
Iustin Pop committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Ganeti small utilities
23

Iustin Pop's avatar
Iustin Pop committed
24
25
26
27
28
29
30
"""


import sys
import os
import sha
import time
31
import subprocess
Iustin Pop's avatar
Iustin Pop committed
32
33
34
35
import re
import socket
import tempfile
import shutil
36
import errno
37
import pwd
Guido Trotter's avatar
Guido Trotter committed
38
import itertools
39
40
import select
import fcntl
41
import resource
42
import logging
Michael Hanselmann's avatar
Michael Hanselmann committed
43
import signal
44
45

from cStringIO import StringIO
Iustin Pop's avatar
Iustin Pop committed
46
47

from ganeti import errors
Iustin Pop's avatar
Iustin Pop committed
48
from ganeti import constants
Iustin Pop's avatar
Iustin Pop committed
49

50

Iustin Pop's avatar
Iustin Pop committed
51
52
53
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')

54
debug = False
55
debug_locks = False
56
no_fork = False
57

58

Iustin Pop's avatar
Iustin Pop committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class RunResult(object):
  """Simple class for holding the result of running external programs.

  Instance variables:
    exit_code: the exit code of the program, or None (if the program
               didn't exit())
    signal: numeric signal that caused the program to finish, or None
            (if the program wasn't terminated by a signal)
    stdout: the standard output of the program
    stderr: the standard error of the program
    failed: a Boolean value which is True in case the program was
            terminated by a signal or exited with a non-zero exit code
    fail_reason: a string detailing the termination reason

  """
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
               "failed", "fail_reason", "cmd"]


Iustin Pop's avatar
Iustin Pop committed
78
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
Iustin Pop's avatar
Iustin Pop committed
79
80
    self.cmd = cmd
    self.exit_code = exit_code
Iustin Pop's avatar
Iustin Pop committed
81
    self.signal = signal_
Iustin Pop's avatar
Iustin Pop committed
82
83
    self.stdout = stdout
    self.stderr = stderr
Iustin Pop's avatar
Iustin Pop committed
84
    self.failed = (signal_ is not None or exit_code != 0)
Iustin Pop's avatar
Iustin Pop committed
85
86
87
88
89
90
91
92

    if self.signal is not None:
      self.fail_reason = "terminated by signal %s" % self.signal
    elif self.exit_code is not None:
      self.fail_reason = "exited with exit code %s" % self.exit_code
    else:
      self.fail_reason = "unable to determine termination reason"

93
94
95
    if self.failed:
      logging.debug("Command '%s' failed (%s); output: %s",
                    self.cmd, self.fail_reason, self.output)
96

Iustin Pop's avatar
Iustin Pop committed
97
98
99
100
101
102
103
104
105
  def _GetOutput(self):
    """Returns the combined stdout and stderr for easier usage.

    """
    return self.stdout + self.stderr

  output = property(_GetOutput, None, None, "Return full output")


106
def RunCmd(cmd, env=None):
Iustin Pop's avatar
Iustin Pop committed
107
108
109
110
111
  """Execute a (shell) command.

  The command should not read from its standard input, as it will be
  closed.

112
113
  @param cmd: Command to run
  @type  cmd: string or list
114
115
  @param env: Additional environment
  @type env: dict
116
117
  @return: `RunResult` instance
  @rtype: RunResult
Iustin Pop's avatar
Iustin Pop committed
118
119

  """
120
121
122
  if no_fork:
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")

Iustin Pop's avatar
Iustin Pop committed
123
124
  if isinstance(cmd, list):
    cmd = [str(val) for val in cmd]
125
126
127
128
129
    strcmd = " ".join(cmd)
    shell = False
  else:
    strcmd = cmd
    shell = True
130
  logging.debug("RunCmd '%s'", strcmd)
131
132
133
134
135
136

  cmd_env = os.environ.copy()
  cmd_env["LC_ALL"] = "C"
  if env is not None:
    cmd_env.update(env)

137
  poller = select.poll()
138
139
140
141
  child = subprocess.Popen(cmd, shell=shell,
                           stderr=subprocess.PIPE,
                           stdout=subprocess.PIPE,
                           stdin=subprocess.PIPE,
142
                           close_fds=True, env=cmd_env)
143
144

  child.stdin.close()
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
  poller.register(child.stdout, select.POLLIN)
  poller.register(child.stderr, select.POLLIN)
  out = StringIO()
  err = StringIO()
  fdmap = {
    child.stdout.fileno(): (out, child.stdout),
    child.stderr.fileno(): (err, child.stderr),
    }
  for fd in fdmap:
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)

  while fdmap:
    for fd, event in poller.poll():
      if event & select.POLLIN or event & select.POLLPRI:
        data = fdmap[fd][1].read()
        # no data from read signifies EOF (the same as POLLHUP)
        if not data:
          poller.unregister(fd)
          del fdmap[fd]
          continue
        fdmap[fd][0].write(data)
      if (event & select.POLLNVAL or event & select.POLLHUP or
          event & select.POLLERR):
        poller.unregister(fd)
        del fdmap[fd]

  out = out.getvalue()
  err = err.getvalue()
Iustin Pop's avatar
Iustin Pop committed
174
175

  status = child.wait()
176
177
  if status >= 0:
    exitcode = status
Iustin Pop's avatar
Iustin Pop committed
178
    signal_ = None
Iustin Pop's avatar
Iustin Pop committed
179
180
  else:
    exitcode = None
Iustin Pop's avatar
Iustin Pop committed
181
    signal_ = -status
Iustin Pop's avatar
Iustin Pop committed
182

Iustin Pop's avatar
Iustin Pop committed
183
  return RunResult(exitcode, signal_, out, err, strcmd)
Iustin Pop's avatar
Iustin Pop committed
184
185
186
187
188
189
190
191
192
193
194
195


def RemoveFile(filename):
  """Remove a file ignoring some errors.

  Remove a file, ignoring non-existing ones or directories. Other
  errors are passed.

  """
  try:
    os.unlink(filename)
  except OSError, err:
196
    if err.errno not in (errno.ENOENT, errno.EISDIR):
Iustin Pop's avatar
Iustin Pop committed
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
      raise


def _FingerprintFile(filename):
  """Compute the fingerprint of a file.

  If the file does not exist, a None will be returned
  instead.

  Args:
    filename - Filename (str)

  """
  if not (os.path.exists(filename) and os.path.isfile(filename)):
    return None

  f = open(filename)

  fp = sha.sha()
  while True:
    data = f.read(4096)
    if not data:
      break

    fp.update(data)

  return fp.hexdigest()


def FingerprintFiles(files):
  """Compute fingerprints for a list of files.

  Args:
    files - array of filenames.  ( [str, ...] )

  Return value:
    dictionary of filename: fingerprint for the files that exist

  """
  ret = {}

  for filename in files:
    cksum = _FingerprintFile(filename)
    if cksum:
      ret[filename] = cksum

  return ret


def CheckDict(target, template, logname=None):
  """Ensure a dictionary has a required set of keys.

  For the given dictionaries `target` and `template`, ensure target
  has all the keys from template. Missing keys are added with values
  from template.

  Args:
    target   - the dictionary to check
    template - template dictionary
    logname  - a caller-chosen string to identify the debug log
               entry; if None, no logging will be done

  Returns value:
    None

  """
  missing = []
  for k in template:
    if k not in target:
      missing.append(k)
      target[k] = template[k]

  if missing and logname:
270
    logging.warning('%s missing keys %s', logname, ', '.join(missing))
Iustin Pop's avatar
Iustin Pop committed
271
272
273
274
275
276
277


def IsProcessAlive(pid):
  """Check if a given pid exists on the system.

  Returns: true or false, depending on if the pid exists or not

278
279
  Remarks: zombie processes treated as not alive, and giving a pid <=
  0 makes the function to return False.
Iustin Pop's avatar
Iustin Pop committed
280
281

  """
282
283
284
  if pid <= 0:
    return False

Iustin Pop's avatar
Iustin Pop committed
285
286
287
  try:
    f = open("/proc/%d/status" % pid)
  except IOError, err:
288
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
Iustin Pop's avatar
Iustin Pop committed
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
      return False

  alive = True
  try:
    data = f.readlines()
    if len(data) > 1:
      state = data[1].split()
      if len(state) > 1 and state[1] == "Z":
        alive = False
  finally:
    f.close()

  return alive


304
305
def ReadPidFile(pidfile):
  """Read the pid from a file.
306

307
308
309
310
311
  @param pidfile: Path to a file containing the pid to be checked
  @type  pidfile: string (filename)
  @return: The process id, if the file exista and contains a valid PID,
           otherwise 0
  @rtype: int
312
313
314
315

  """
  try:
    pf = open(pidfile, 'r')
316
317
318
319
  except EnvironmentError, err:
    if err.errno != errno.ENOENT:
      logging.exception("Can't read pid file?!")
    return 0
320
321
322

  try:
    pid = int(pf.read())
323
  except ValueError, err:
324
    logging.info("Can't parse pid file contents", exc_info=True)
325
    return 0
326

327
  return pid
328
329


Iustin Pop's avatar
Iustin Pop committed
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
def MatchNameComponent(key, name_list):
  """Try to match a name against a list.

  This function will try to match a name like test1 against a list
  like ['test1.example.com', 'test2.example.com', ...]. Against this
  list, 'test1' as well as 'test1.example' will match, but not
  'test1.ex'. A multiple match will be considered as no match at all
  (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).

  Args:
    key: the name to be searched
    name_list: the list of strings against which to search the key

  Returns:
    None if there is no match *or* if there are multiple matches
    otherwise the element from the list which matches

  """
  mo = re.compile("^%s(\..*)?$" % re.escape(key))
  names_filtered = [name for name in name_list if mo.match(name) is not None]
  if len(names_filtered) != 1:
    return None
  return names_filtered[0]


355
class HostInfo:
356
  """Class implementing resolver and hostname functionality
357
358

  """
359
  def __init__(self, name=None):
360
361
    """Initialize the host name object.

362
363
    If the name argument is not passed, it will use this system's
    name.
364
365

    """
366
367
368
369
370
    if name is None:
      name = self.SysName()

    self.query = name
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
371
372
    self.ip = self.ipaddrs[0]

373
374
375
376
377
378
  def ShortName(self):
    """Returns the hostname without domain.

    """
    return self.name.split('.')[0]

379
380
381
  @staticmethod
  def SysName():
    """Return the current system's name.
382

383
    This is simply a wrapper over socket.gethostname()
Iustin Pop's avatar
Iustin Pop committed
384

385
386
    """
    return socket.gethostname()
Iustin Pop's avatar
Iustin Pop committed
387

388
389
390
  @staticmethod
  def LookupHostname(hostname):
    """Look up hostname
Iustin Pop's avatar
Iustin Pop committed
391

392
393
394
395
396
397
398
399
400
401
402
403
404
    Args:
      hostname: hostname to look up

    Returns:
      a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
      in case of errors in resolving, we raise a ResolverError

    """
    try:
      result = socket.gethostbyname_ex(hostname)
    except socket.gaierror, err:
      # hostname not found in DNS
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
Iustin Pop's avatar
Iustin Pop committed
405

406
    return result
Iustin Pop's avatar
Iustin Pop committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426


def ListVolumeGroups():
  """List volume groups and their size

  Returns:
     Dictionary with keys volume name and values the size of the volume

  """
  command = "vgs --noheadings --units m --nosuffix -o name,size"
  result = RunCmd(command)
  retval = {}
  if result.failed:
    return retval

  for line in result.stdout.splitlines():
    try:
      name, size = line.split()
      size = int(float(size))
    except (IndexError, ValueError), err:
427
      logging.error("Invalid output from vgs (%s): %s", err, line)
Iustin Pop's avatar
Iustin Pop committed
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
      continue

    retval[name] = size

  return retval


def BridgeExists(bridge):
  """Check whether the given bridge exists in the system

  Returns:
     True if it does, false otherwise.

  """
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)


def NiceSort(name_list):
  """Sort a list of strings based on digit and non-digit groupings.

  Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
  sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].

  The sort algorithm breaks each name in groups of either only-digits
  or no-digits. Only the first eight such groups are considered, and
  after that we just use what's left of the string.

  Return value
    - a copy of the list sorted according to our algorithm

  """
  _SORTER_BASE = "(\D+|\d+)"
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE)
  _SORTER_RE = re.compile(_SORTER_FULL)
  _SORTER_NODIGIT = re.compile("^\D*$")
  def _TryInt(val):
    """Attempts to convert a variable to integer."""
    if val is None or _SORTER_NODIGIT.match(val):
      return val
    rval = int(val)
    return rval

  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
             for name in name_list]
  to_sort.sort()
  return [tup[1] for tup in to_sort]


def TryConvert(fn, val):
  """Try to convert a value ignoring errors.

  This function tries to apply function `fn` to `val`. If no
  ValueError or TypeError exceptions are raised, it will return the
  result, else it will return the original value. Any other exceptions
  are propagated to the caller.

  """
  try:
    nv = fn(val)
  except (ValueError, TypeError), err:
    nv = val
  return nv


def IsValidIP(ip):
  """Verifies the syntax of an IP address.

  This function checks if the ip address passes is valid or not based
  on syntax (not ip range, class calculations or anything).

  """
  unit = "(0|[1-9]\d{0,2})"
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)


def IsValidShellParam(word):
  """Verifies is the given word is safe from the shell's p.o.v.

  This means that we can pass this to a command via the shell and be
  sure that it doesn't alter the command line and is passed as such to
  the actual command.

  Note that we are overly restrictive here, in order to be on the safe
  side.

  """
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))


def BuildShellCmd(template, *args):
  """Build a safe shell command line from the given arguments.

  This function will check all arguments in the args list so that they
  are valid shell parameters (i.e. they don't contain shell
  metacharaters). If everything is ok, it will return the result of
  template % args.

  """
  for word in args:
    if not IsValidShellParam(word):
531
532
      raise errors.ProgrammerError("Shell argument '%s' contains"
                                   " invalid characters" % word)
Iustin Pop's avatar
Iustin Pop committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
  return template % args


def FormatUnit(value):
  """Formats an incoming number of MiB with the appropriate unit.

  Value needs to be passed as a numeric type. Return value is always a string.

  """
  if value < 1024:
    return "%dM" % round(value, 0)

  elif value < (1024 * 1024):
    return "%0.1fG" % round(float(value) / 1024, 1)

  else:
    return "%0.1fT" % round(float(value) / 1024 / 1024, 1)


def ParseUnit(input_string):
  """Tries to extract number and scale from the given string.

  Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
  is specified, it defaults to MiB. Return value is always an int in MiB.

  """
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
  if not m:
561
    raise errors.UnitParseError("Invalid format")
Iustin Pop's avatar
Iustin Pop committed
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581

  value = float(m.groups()[0])

  unit = m.groups()[1]
  if unit:
    lcunit = unit.lower()
  else:
    lcunit = 'm'

  if lcunit in ('m', 'mb', 'mib'):
    # Value already in MiB
    pass

  elif lcunit in ('g', 'gb', 'gib'):
    value *= 1024

  elif lcunit in ('t', 'tb', 'tib'):
    value *= 1024 * 1024

  else:
582
    raise errors.UnitParseError("Unknown unit: %s" % unit)
Iustin Pop's avatar
Iustin Pop committed
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633

  # Make sure we round up
  if int(value) < value:
    value += 1

  # Round up to the next multiple of 4
  value = int(value)
  if value % 4:
    value += 4 - value % 4

  return value


def AddAuthorizedKey(file_name, key):
  """Adds an SSH public key to an authorized_keys file.

  Args:
    file_name: Path to authorized_keys file
    key: String containing key
  """
  key_fields = key.split()

  f = open(file_name, 'a+')
  try:
    nl = True
    for line in f:
      # Ignore whitespace changes
      if line.split() == key_fields:
        break
      nl = line.endswith('\n')
    else:
      if not nl:
        f.write("\n")
      f.write(key.rstrip('\r\n'))
      f.write("\n")
      f.flush()
  finally:
    f.close()


def RemoveAuthorizedKey(file_name, key):
  """Removes an SSH public key from an authorized_keys file.

  Args:
    file_name: Path to authorized_keys file
    key: String containing key
  """
  key_fields = key.split()

  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
634
    out = os.fdopen(fd, 'w')
Iustin Pop's avatar
Iustin Pop committed
635
    try:
636
637
638
639
640
641
      f = open(file_name, 'r')
      try:
        for line in f:
          # Ignore whitespace changes while comparing lines
          if line.split() != key_fields:
            out.write(line)
642
643
644
645
646
647
648
649
650
651
652
653

        out.flush()
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise


654
655
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
  """Sets the name of an IP address and hostname in /etc/hosts.
656
657

  """
658
659
660
  # Ensure aliases are unique
  aliases = UniqueSequence([hostname] + aliases)[1:]

661
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
662
  try:
663
664
665
666
667
668
669
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        written = False
        for line in f:
          fields = line.split()
670
          if fields and not fields[0].startswith('#') and ip == fields[0]:
671
672
673
            continue
          out.write(line)

674
        out.write("%s\t%s" % (ip, hostname))
675
676
677
678
679
        if aliases:
          out.write(" %s" % ' '.join(aliases))
        out.write('\n')

        out.flush()
680
        os.fsync(out)
681
682
683
684
685
686
687
688
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise
689
690


691
692
693
694
695
696
697
698
def AddHostToEtcHosts(hostname):
  """Wrapper around SetEtcHostsEntry.

  """
  hi = HostInfo(name=hostname)
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])


699
def RemoveEtcHostsEntry(file_name, hostname):
700
  """Removes a hostname from /etc/hosts.
701

702
  IP addresses without names are removed from the file.
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
  """
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        for line in f:
          fields = line.split()
          if len(fields) > 1 and not fields[0].startswith('#'):
            names = fields[1:]
            if hostname in names:
              while hostname in names:
                names.remove(hostname)
              if names:
718
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
719
720
721
              continue

          out.write(line)
722
723

        out.flush()
724
        os.fsync(out)
725
726
727
        os.rename(tmpname, file_name)
      finally:
        f.close()
Iustin Pop's avatar
Iustin Pop committed
728
    finally:
729
730
731
732
      out.close()
  except:
    RemoveFile(tmpname)
    raise
Iustin Pop's avatar
Iustin Pop committed
733
734


735
736
737
738
739
740
741
742
743
def RemoveHostFromEtcHosts(hostname):
  """Wrapper around RemoveEtcHostsEntry.

  """
  hi = HostInfo(name=hostname)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())


Iustin Pop's avatar
Iustin Pop committed
744
745
746
747
748
749
750
def CreateBackup(file_name):
  """Creates a backup of a file.

  Returns: the path to the newly created backup file.

  """
  if not os.path.isfile(file_name):
751
752
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
                                file_name)
Iustin Pop's avatar
Iustin Pop committed
753

754
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
Iustin Pop's avatar
Iustin Pop committed
755
  dir_name = os.path.dirname(file_name)
756
757
758

  fsrc = open(file_name, 'rb')
  try:
Iustin Pop's avatar
Iustin Pop committed
759
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
760
761
762
763
764
765
766
767
    fdst = os.fdopen(fd, 'wb')
    try:
      shutil.copyfileobj(fsrc, fdst)
    finally:
      fdst.close()
  finally:
    fsrc.close()

Iustin Pop's avatar
Iustin Pop committed
768
769
770
771
772
  return backup_name


def ShellQuote(value):
  """Quotes shell argument according to POSIX.
773

Iustin Pop's avatar
Iustin Pop committed
774
775
776
777
778
779
780
781
782
783
784
785
  """
  if _re_shell_unquoted.match(value):
    return value
  else:
    return "'%s'" % value.replace("'", "'\\''")


def ShellQuoteArgs(args):
  """Quotes all given shell arguments and concatenates using spaces.

  """
  return ' '.join([ShellQuote(i) for i in args])
786
787


788
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
789
790
  """Simple ping implementation using TCP connect(2).

791
792
793
794
795
796
797
798
  Try to do a TCP connect(2) from an optional source IP to the
  specified target IP and the specified target port. If the optional
  parameter live_port_needed is set to true, requires the remote end
  to accept the connection. The timeout is specified in seconds and
  defaults to 10 seconds. If the source optional argument is not
  passed, the source address selection is left to the kernel,
  otherwise we try to connect using the passed address (failures to
  bind other than EADDRNOTAVAIL will be ignored).
799
800
801
802
803
804

  """
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

  sucess = False

805
806
807
808
809
810
  if source is not None:
    try:
      sock.bind((source, 0))
    except socket.error, (errcode, errstring):
      if errcode == errno.EADDRNOTAVAIL:
        success = False
811
812
813
814
815
816
817
818
819
820

  sock.settimeout(timeout)

  try:
    sock.connect((target, port))
    sock.close()
    success = True
  except socket.timeout:
    success = False
  except socket.error, (errcode, errstring):
821
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
822
823

  return success
824
825


826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
def OwnIpAddress(address):
  """Check if the current host has the the given IP address.

  Currently this is done by tcp-pinging the address from the loopback
  address.

  @type address: string
  @param address: the addres to check
  @rtype: bool

  """
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
                 source=constants.LOCALHOST_IP_ADDRESS)


841
842
843
844
def ListVisibleFiles(path):
  """Returns a list of all visible files in a directory.

  """
845
846
847
  files = [i for i in os.listdir(path) if not i.startswith(".")]
  files.sort()
  return files
848
849


850
851
852
853
854
855
def GetHomeDir(user, default=None):
  """Try to get the homedir of the given user.

  The user can be passed either as a string (denoting the name) or as
  an integer (denoting the user id). If the user is not found, the
  'default' argument is returned, which defaults to None.
856
857
858

  """
  try:
859
860
861
862
863
864
865
    if isinstance(user, basestring):
      result = pwd.getpwnam(user)
    elif isinstance(user, (int, long)):
      result = pwd.getpwuid(user)
    else:
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
                                   type(user))
866
867
868
  except KeyError:
    return default
  return result.pw_dir
869
870


871
def NewUUID():
872
873
874
875
876
877
878
879
  """Returns a random UUID.

  """
  f = open("/proc/sys/kernel/random/uuid", "r")
  try:
    return f.read(128).rstrip("\n")
  finally:
    f.close()
Iustin Pop's avatar
Iustin Pop committed
880
881


882
883
884
885
886
887
888
889
890
891
def GenerateSecret():
  """Generates a random secret.

  This will generate a pseudo-random secret, and return its sha digest
  (so that it can be used where an ASCII string is needed).

  """
  return sha.new(os.urandom(64)).hexdigest()


892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
def ReadFile(file_name, size=None):
  """Reads a file.

  @type size: None or int
  @param size: Read at most size bytes

  """
  f = open(file_name, "r")
  try:
    if size is None:
      return f.read()
    else:
      return f.read(size)
  finally:
    f.close()


Iustin Pop's avatar
Iustin Pop committed
909
910
def WriteFile(file_name, fn=None, data=None,
              mode=None, uid=-1, gid=-1,
911
              atime=None, mtime=None, close=True,
912
              dry_run=False, backup=False,
913
              prewrite=None, postwrite=None):
Iustin Pop's avatar
Iustin Pop committed
914
915
916
917
918
919
920
921
922
923
924
925
926
  """(Over)write a file atomically.

  The file_name and either fn (a function taking one argument, the
  file descriptor, and which should write the data to it) or data (the
  contents of the file) must be passed. The other arguments are
  optional and allow setting the file mode, owner and group, and the
  mtime/atime of the file.

  If the function doesn't raise an exception, it has succeeded and the
  target file has the new contents. If the file has raised an
  exception, an existing target file should be unmodified and the
  temporary file should be removed.

927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
  Args:
    file_name: New filename
    fn: Content writing function, called with file descriptor as parameter
    data: Content as string
    mode: File mode
    uid: Owner
    gid: Group
    atime: Access time
    mtime: Modification time
    close: Whether to close file after writing it
    prewrite: Function object called before writing content
    postwrite: Function object called after writing content

  Returns:
    None if "close" parameter evaluates to True, otherwise file descriptor.

Iustin Pop's avatar
Iustin Pop committed
943
  """
944
  if not os.path.isabs(file_name):
Iustin Pop's avatar
Iustin Pop committed
945
946
947
948
949
950
951
952
953
954
    raise errors.ProgrammerError("Path passed to WriteFile is not"
                                 " absolute: '%s'" % file_name)

  if [fn, data].count(None) != 1:
    raise errors.ProgrammerError("fn or data required")

  if [atime, mtime].count(None) == 1:
    raise errors.ProgrammerError("Both atime and mtime must be either"
                                 " set or None")

Michael Hanselmann's avatar
Michael Hanselmann committed
955
956
  if backup and not dry_run and os.path.isfile(file_name):
    CreateBackup(file_name)
Iustin Pop's avatar
Iustin Pop committed
957
958
959
960
961
962
963
964
965
966

  dir_name, base_name = os.path.split(file_name)
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
  # here we need to make sure we remove the temp file, if any error
  # leaves it in place
  try:
    if uid != -1 or gid != -1:
      os.chown(new_name, uid, gid)
    if mode:
      os.chmod(new_name, mode)
967
968
    if callable(prewrite):
      prewrite(fd)
Iustin Pop's avatar
Iustin Pop committed
969
970
971
972
    if data is not None:
      os.write(fd, data)
    else:
      fn(fd)
973
974
    if callable(postwrite):
      postwrite(fd)
Iustin Pop's avatar
Iustin Pop committed
975
976
977
    os.fsync(fd)
    if atime is not None and mtime is not None:
      os.utime(new_name, (atime, mtime))
Michael Hanselmann's avatar
Michael Hanselmann committed
978
979
    if not dry_run:
      os.rename(new_name, file_name)
Iustin Pop's avatar
Iustin Pop committed
980
  finally:
981
982
983
984
985
    if close:
      os.close(fd)
      result = None
    else:
      result = fd
Iustin Pop's avatar
Iustin Pop committed
986
    RemoveFile(new_name)
Guido Trotter's avatar
Guido Trotter committed
987

988
989
  return result

Guido Trotter's avatar
Guido Trotter committed
990

991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
def FirstFree(seq, base=0):
  """Returns the first non-existing integer from seq.

  The seq argument should be a sorted list of positive integers. The
  first time the index of an element is smaller than the element
  value, the index will be returned.

  The base argument is used to start at a different offset,
  i.e. [3, 4, 6] with offset=3 will return 5.

  Example: [0, 1, 3] will return 2.

  """
  for idx, elem in enumerate(seq):
    assert elem >= base, "Passed element is higher than base offset"
    if elem > idx + base:
      # idx is not used
      return idx + base
  return None


Guido Trotter's avatar
Guido Trotter committed
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
def all(seq, pred=bool):
  "Returns True if pred(x) is True for every element in the iterable"
  for elem in itertools.ifilterfalse(pred, seq):
    return False
  return True


def any(seq, pred=bool):
  "Returns True if pred(x) is True for at least one element in the iterable"
  for elem in itertools.ifilter(pred, seq):
    return True
  return False
1024
1025
1026
1027
1028
1029
1030
1031
1032


def UniqueSequence(seq):
  """Returns a list with unique elements.

  Element order is preserved.
  """
  seen = set()
  return [i for i in seq if i not in seen and not seen.add(i)]
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042


def IsValidMac(mac):
  """Predicate to check if a MAC address is valid.

  Checks wether the supplied MAC address is formally correct, only
  accepts colon separated format.
  """
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
  return mac_check.match(mac) is not None
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052


def TestDelay(duration):
  """Sleep for a fixed amount of time.

  """
  if duration < 0:
    return False
  time.sleep(duration)
  return True
1053
1054


1055
def Daemonize(logfile, noclose_fds=None):
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
  """Daemonize the current process.

  This detaches the current process from the controlling terminal and
  runs it in the background as a daemon.

  """
  UMASK = 077
  WORKDIR = "/"
  # Default maximum for the number of available file descriptors.
  if 'SC_OPEN_MAX' in os.sysconf_names:
    try:
      MAXFD = os.sysconf('SC_OPEN_MAX')
      if MAXFD < 0:
        MAXFD = 1024
    except OSError:
      MAXFD = 1024
  else:
    MAXFD = 1024

  # this might fail
  pid = os.fork()
  if (pid == 0):  # The first child.
    os.setsid()
    # this might fail
    pid = os.fork() # Fork a second child.
    if (pid == 0):  # The second child.
      os.chdir(WORKDIR)
      os.umask(UMASK)
    else:
      # exit() or _exit()?  See below.
      os._exit(0) # Exit parent (the first child) of the second child.
  else:
    os._exit(0) # Exit parent of the first child.
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
  if (maxfd == resource.RLIM_INFINITY):
    maxfd = MAXFD

  # Iterate through and close all file descriptors.
  for fd in range(0, maxfd):
1095
1096
    if noclose_fds and fd in noclose_fds:
      continue
1097
1098
1099
1100
1101
1102
1103
1104
1105
    try:
      os.close(fd)
    except OSError: # ERROR, fd wasn't open to begin with (ignored)
      pass
  os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
  # Duplicate standard input to standard output and standard error.
  os.dup2(0, 1)     # standard output (1)
  os.dup2(0, 2)     # standard error (2)
  return 0
1106
1107


Iustin Pop's avatar
Iustin Pop committed
1108
def DaemonPidFileName(name):
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
  """Compute a ganeti pid file absolute path, given the daemon name.

  """
  return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)


def WritePidFile(name):
  """Write the current process pidfile.

  The file will be written to constants.RUN_GANETI_DIR/name.pid

  """
  pid = os.getpid()
Iustin Pop's avatar
Iustin Pop committed
1122
  pidfilename = DaemonPidFileName(name)
1123
  if IsProcessAlive(ReadPidFile(pidfilename)):
1124
    raise errors.GenericError("%s contains a live process" % pidfilename)
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135

  WriteFile(pidfilename, data="%d\n" % pid)


def RemovePidFile(name):
  """Remove the current process pidfile.

  Any errors are ignored.

  """
  pid = os.getpid()
Iustin Pop's avatar
Iustin Pop committed
1136
  pidfilename = DaemonPidFileName(name)
1137
1138
1139
1140
1141
1142
1143
  # TODO: we could check here that the file contains our pid
  try:
    RemoveFile(pidfilename)
  except:
    pass


Iustin Pop's avatar
Iustin Pop committed
1144
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30):
Iustin Pop's avatar
Iustin Pop committed
1145
1146
1147
1148
  """Kill a process given by its pid.

  @type pid: int
  @param pid: The PID to terminate.
Iustin Pop's avatar
Iustin Pop committed
1149
1150
  @type signal_: int
  @param signal_: The signal to send, by default SIGTERM
Iustin Pop's avatar
Iustin Pop committed
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
  @type timeout: int
  @param timeout: The timeout after which, if the process is still alive,
                  a SIGKILL will be sent. If not positive, no such checking
                  will be done

  """
  if pid <= 0:
    # kill with pid=0 == suicide
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)

  if not IsProcessAlive(pid):
    return
Iustin Pop's avatar
Iustin Pop committed
1163
  os.kill(pid, signal_)
Iustin Pop's avatar
Iustin Pop committed
1164
1165
1166
1167
1168
1169
1170
1171
1172
  if timeout <= 0:
    return
  end = time.time() + timeout
  while time.time() < end and IsProcessAlive(pid):
    time.sleep(0.1)
  if IsProcessAlive(pid):
    os.kill(pid, signal.SIGKILL)


1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
def FindFile(name, search_path, test=os.path.exists):
  """Look for a filesystem object in a given path.

  This is an abstract method to search for filesystem object (files,
  dirs) under a given search path.

  Args:
    - name: the name to look for
    - search_path: list of directory names
    - test: the test which the full path must satisfy
      (defaults to os.path.exists)

  Returns:
    - full path to the item if found
    - None otherwise

  """
  for dir_name in search_path:
    item_name = os.path.sep.join([dir_name, name])
    if test(item_name):
      return item_name
  return None
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210


def CheckVolumeGroupSize(vglist, vgname, minsize):
  """Checks if the volume group list is valid.

  A non-None return value means there's an error, and the return value
  is the error message.

  """
  vgsize = vglist.get(vgname, None)
  if vgsize is None:
    return "volume group '%s' missing" % vgname
  elif vgsize < minsize:
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
            (vgname, minsize, vgsize))
  return None
1211
1212


1213
def SplitTime(value):
1214
1215
  """Splits time as floating point number into a tuple.

1216
1217
1218
  @param value: Time in seconds
  @type value: int or float
  @return: Tuple containing (seconds, microseconds)
1219
1220

  """
1221
1222
1223
1224
1225
1226
1227
1228
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)

  assert 0 <= seconds, \
    "Seconds must be larger than or equal to 0, but are %s" % seconds
  assert 0 <= microseconds <= 999999, \
    "Microseconds must be 0-999999, but are %s" % microseconds

  return (int(seconds), int(microseconds))
1229
1230
1231
1232
1233


def MergeTime(timetuple):
  """Merges a tuple into time as a floating point number.

1234
  @param timetuple: Time as tuple, (seconds, microseconds)
1235
1236
1237
1238
  @type timetuple: tuple
  @return: Time as a floating point number expressed in seconds

  """
1239
  (seconds, microseconds) = timetuple
1240

1241
1242
1243
1244
  assert 0 <= seconds, \
    "Seconds must be larger than or equal to 0, but are %s" % seconds
  assert 0 <= microseconds <= 999999, \
    "Microseconds must be 0-999999, but are %s" % microseconds
1245

1246
  return float(seconds) + (float(microseconds) * 0.000001)
1247
1248


1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
def GetNodeDaemonPort():
  """Get the node daemon port for this cluster.

  Note that this routine does not read a ganeti-specific file, but
  instead uses socket.getservbyname to allow pre-customization of
  this parameter outside of Ganeti.

  """
  try:
    port = socket.getservbyname("ganeti-noded", "tcp")
  except socket.error:
    port = constants.DEFAULT_NODED_PORT

  return port


def GetNodeDaemonPassword():
  """Get the node password for the cluster.

  """
  return ReadFile(constants.CLUSTER_PASSWORD_FILE)


1272
1273
1274
1275
1276
1277
1278
def LockedMethod(fn):
  """Synchronized object access decorator.

  This decorator is intended to protect access to an object using the
  object's own lock which is hardcoded to '_lock'.

  """
1279
1280
1281
1282
  def _LockDebug(*args, **kwargs):
    if debug_locks:
      logging.debug(*args, **kwargs)

1283
1284
1285
  def wrapper(self, *args, **kwargs):
    assert hasattr(self, '_lock')
    lock = self._lock
1286
    _LockDebug("Waiting for %s", lock)
1287
1288
    lock.acquire()
    try:
1289
      _LockDebug("Acquired %s", lock)
1290
1291
      result = fn(self, *args, **kwargs)
    finally:
1292
      _LockDebug("Releasing %s", lock)
1293
      lock.release()
1294
      _LockDebug("Released %s", lock)
1295
1296
    return result
  return wrapper
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308


def LockFile(fd):
  """Locks a file using POSIX locks.

  """
  try:
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  except IOError, err:
    if err.errno == errno.EAGAIN:
      raise errors.LockError("File already locked")
    raise
Michael Hanselmann's avatar
Michael Hanselmann committed
1309
1310


1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
class FileLock(object):
  """Utility class for file locks.

  """
  def __init__(self, filename):
    self.filename = filename
    self.fd = open(self.filename, "w")

  def __del__(self):
    self.Close()

  def Close(self):
    if self.fd:
      self.fd.close()
      self.fd = None

1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
  def _flock(self, flag, blocking, timeout, errmsg):
    """Wrapper for fcntl.flock.

    @type flag: int
    @param flag: Operation flag
    @type blocking: bool
    @param blocking: Whether the operation should be done in blocking mode.
    @type timeout: None or float
    @param timeout: For how long the operation should be retried (implies
                    non-blocking mode).
    @type errmsg: string
    @param errmsg: Error message in case operation fails.

    """
1341
    assert self.fd, "Lock was closed"
1342
1343
    assert timeout is None or timeout >= 0, \
      "If specified, timeout must be positive"
1344

1345
    if timeout is not None:
1346
      flag |= fcntl.LOCK_NB
1347
      timeout_end = time.time() + timeout
1348

1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
    # Blocking doesn't have effect with timeout
    elif not blocking:
      flag |= fcntl.LOCK_NB
      timeout_end = None

    retry = True
    while retry:
      try:
        fcntl.flock(self.fd, flag)
        retry = False
      except IOError, err:
        if err.errno in (errno.EAGAIN, ):
          if timeout_end is not None and time.time() < timeout_end:
            # Wait before trying again
            time.sleep(max(0.1, min(1.0, timeout)))
          else:
            raise errors.LockError(errmsg)
        else:
          logging.exception("fcntl.flock failed")
          raise

  def Exclusive(self, blocking=False, timeout=None):
1371
1372
1373
    """Locks the file in exclusive mode.

    """
1374
    self._flock(fcntl.LOCK_EX, blocking, timeout,
1375
1376
                "Failed to lock %s in exclusive mode" % self.filename)

1377
  def Shared(self, blocking=False, timeout=None):
1378
1379
1380
    """Locks the file in shared mode.

    """
1381
    self._flock(fcntl.LOCK_SH, blocking, timeout,
1382
1383
                "Failed to lock %s in shared mode" % self.filename)

1384
  def Unlock(self, blocking=True, timeout=None):
1385
1386
1387
1388
1389
1390
1391
    """Unlocks the file.

    According to "man flock", unlocking can also be a nonblocking operation:
    "To make a non-blocking request, include LOCK_NB with any of the above
    operations"

    """
1392
    self._flock(fcntl.LOCK_UN, blocking, timeout,
1393
1394
1395
                "Failed to unlock %s" % self.filename)


Michael Hanselmann's avatar
Michael Hanselmann committed
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
class SignalHandler(object):
  """Generic signal handler class.

  It automatically restores the original handler when deconstructed or when
  Reset() is called. You can either pass your own handler function in or query
  the "called" attribute to detect whether the signal was sent.

  """
  def __init__(self, signum):
    """Constructs a new SignalHandler instance.

    @param signum: Single signal number or set of signal numbers

    """
    if isinstance(signum, (int, long)):
      self.signum = set([signum])
    else:
      self.signum = set(signum)

    self.called = False

    self._previous = {}
    try:
      for signum in self.signum:
        # Setup handler
        prev_handler = signal.signal(signum, self._HandleSignal)
        try:
          self._previous[signum] = prev_handler
        except:
          # Restore previous handler
          signal.signal(signum, prev_handler)
          raise
    except:
      # Reset all handlers
      self.Reset()
      # Here we have a race condition: a handler may have already been called,
      # but there's not much we can do about it at this point.
      raise

  def __del__(self):
    self.Reset()

  def Reset(self):
    """Restore previous handler.

    """
    for signum, prev_handler in self._previous.items():
      signal.signal(signum, prev_handler)
      # If successful, remove from dict
      del self._previous[signum]

  def Clear(self):
    """Unsets "called" flag.

    This function can be used in case a signal may arrive several times.

    """
    self.called = False

  def _HandleSignal(self, signum, frame):
    """Actual signal handling function.

    """
    # This is not nice and not absolutely atomic, but it appears to be the only
    # solution in Python -- there are no atomic types.
    self.called = True