utils.py 72 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#
Iustin Pop's avatar
Iustin Pop committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22 23 24 25
"""Ganeti utility module.

This module holds functions that can be used in both daemons (all) and
the command line scripts.
26

Iustin Pop's avatar
Iustin Pop committed
27 28 29 30 31
"""


import os
import time
32
import subprocess
Iustin Pop's avatar
Iustin Pop committed
33 34 35 36
import re
import socket
import tempfile
import shutil
37
import errno
38
import pwd
Guido Trotter's avatar
Guido Trotter committed
39
import itertools
40 41
import select
import fcntl
42
import resource
43
import logging
44
import logging.handlers
Michael Hanselmann's avatar
Michael Hanselmann committed
45
import signal
46 47

from cStringIO import StringIO
Iustin Pop's avatar
Iustin Pop committed
48

49 50 51 52 53 54
try:
  from hashlib import sha1
except ImportError:
  import sha
  sha1 = sha.new

Iustin Pop's avatar
Iustin Pop committed
55
from ganeti import errors
Iustin Pop's avatar
Iustin Pop committed
56
from ganeti import constants
Iustin Pop's avatar
Iustin Pop committed
57

58

Iustin Pop's avatar
Iustin Pop committed
59 60 61
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')

62
debug_locks = False
63 64

#: when set to True, L{RunCmd} is disabled
65
no_fork = False
66

67 68
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"

69

Iustin Pop's avatar
Iustin Pop committed
70
class RunResult(object):
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
  """Holds the result of running external programs.

  @type exit_code: int
  @ivar exit_code: the exit code of the program, or None (if the program
      didn't exit())
  @type signal: int or None
  @ivar signal: the signal that caused the program to finish, or None
      (if the program wasn't terminated by a signal)
  @type stdout: str
  @ivar stdout: the standard output of the program
  @type stderr: str
  @ivar stderr: the standard error of the program
  @type failed: boolean
  @ivar failed: True in case the program was
      terminated by a signal or exited with a non-zero exit code
  @ivar fail_reason: a string detailing the termination reason
Iustin Pop's avatar
Iustin Pop committed
87 88 89 90 91 92

  """
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
               "failed", "fail_reason", "cmd"]


Iustin Pop's avatar
Iustin Pop committed
93
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
Iustin Pop's avatar
Iustin Pop committed
94 95
    self.cmd = cmd
    self.exit_code = exit_code
Iustin Pop's avatar
Iustin Pop committed
96
    self.signal = signal_
Iustin Pop's avatar
Iustin Pop committed
97 98
    self.stdout = stdout
    self.stderr = stderr
Iustin Pop's avatar
Iustin Pop committed
99
    self.failed = (signal_ is not None or exit_code != 0)
Iustin Pop's avatar
Iustin Pop committed
100 101 102 103 104 105 106 107

    if self.signal is not None:
      self.fail_reason = "terminated by signal %s" % self.signal
    elif self.exit_code is not None:
      self.fail_reason = "exited with exit code %s" % self.exit_code
    else:
      self.fail_reason = "unable to determine termination reason"

108 109 110
    if self.failed:
      logging.debug("Command '%s' failed (%s); output: %s",
                    self.cmd, self.fail_reason, self.output)
111

Iustin Pop's avatar
Iustin Pop committed
112 113 114 115 116 117 118 119 120
  def _GetOutput(self):
    """Returns the combined stdout and stderr for easier usage.

    """
    return self.stdout + self.stderr

  output = property(_GetOutput, None, None, "Return full output")


Guido Trotter's avatar
Guido Trotter committed
121
def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
Iustin Pop's avatar
Iustin Pop committed
122 123 124 125 126
  """Execute a (shell) command.

  The command should not read from its standard input, as it will be
  closed.

Guido Trotter's avatar
Guido Trotter committed
127
  @type cmd: string or list
128
  @param cmd: Command to run
129
  @type env: dict
130
  @param env: Additional environment
131
  @type output: str
132
  @param output: if desired, the output of the command can be
133 134
      saved in a file instead of the RunResult instance; this
      parameter denotes the file name (if not None)
135 136 137
  @type cwd: string
  @param cwd: if specified, will be used as the working
      directory for the command; the default will be /
Guido Trotter's avatar
Guido Trotter committed
138 139
  @type reset_env: boolean
  @param reset_env: whether to reset or keep the default os environment
140
  @rtype: L{RunResult}
141
  @return: RunResult instance
Michael Hanselmann's avatar
Michael Hanselmann committed
142
  @raise errors.ProgrammerError: if we call this when forks are disabled
Iustin Pop's avatar
Iustin Pop committed
143 144

  """
145 146 147
  if no_fork:
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")

Iustin Pop's avatar
Iustin Pop committed
148 149
  if isinstance(cmd, list):
    cmd = [str(val) for val in cmd]
150 151 152 153 154
    strcmd = " ".join(cmd)
    shell = False
  else:
    strcmd = cmd
    shell = True
155
  logging.debug("RunCmd '%s'", strcmd)
156

Guido Trotter's avatar
Guido Trotter committed
157 158 159 160 161 162
  if not reset_env:
    cmd_env = os.environ.copy()
    cmd_env["LC_ALL"] = "C"
  else:
    cmd_env = {}

163 164 165
  if env is not None:
    cmd_env.update(env)

166 167 168 169 170 171 172 173 174 175 176 177
  try:
    if output is None:
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
    else:
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
      out = err = ""
  except OSError, err:
    if err.errno == errno.ENOENT:
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
                               (strcmd, err))
    else:
      raise
178 179 180 181 182 183 184 185 186 187

  if status >= 0:
    exitcode = status
    signal_ = None
  else:
    exitcode = None
    signal_ = -status

  return RunResult(exitcode, signal_, out, err, strcmd)

188

189
def _RunCmdPipe(cmd, env, via_shell, cwd):
190 191 192 193 194 195 196 197
  """Run a command and return its output.

  @type  cmd: string or list
  @param cmd: Command to run
  @type env: dict
  @param env: The environment to use
  @type via_shell: bool
  @param via_shell: if we should run via the shell
198 199
  @type cwd: string
  @param cwd: the working directory for the program
200 201 202 203
  @rtype: tuple
  @return: (out, err, status)

  """
204
  poller = select.poll()
205
  child = subprocess.Popen(cmd, shell=via_shell,
206 207 208
                           stderr=subprocess.PIPE,
                           stdout=subprocess.PIPE,
                           stdin=subprocess.PIPE,
209 210
                           close_fds=True, env=env,
                           cwd=cwd)
211 212

  child.stdin.close()
213 214 215 216 217 218 219 220 221 222 223 224 225
  poller.register(child.stdout, select.POLLIN)
  poller.register(child.stderr, select.POLLIN)
  out = StringIO()
  err = StringIO()
  fdmap = {
    child.stdout.fileno(): (out, child.stdout),
    child.stderr.fileno(): (err, child.stderr),
    }
  for fd in fdmap:
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)

  while fdmap:
226 227 228 229 230 231 232 233 234 235 236 237
    try:
      pollresult = poller.poll()
    except EnvironmentError, eerr:
      if eerr.errno == errno.EINTR:
        continue
      raise
    except select.error, serr:
      if serr[0] == errno.EINTR:
        continue
      raise

    for fd, event in pollresult:
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
      if event & select.POLLIN or event & select.POLLPRI:
        data = fdmap[fd][1].read()
        # no data from read signifies EOF (the same as POLLHUP)
        if not data:
          poller.unregister(fd)
          del fdmap[fd]
          continue
        fdmap[fd][0].write(data)
      if (event & select.POLLNVAL or event & select.POLLHUP or
          event & select.POLLERR):
        poller.unregister(fd)
        del fdmap[fd]

  out = out.getvalue()
  err = err.getvalue()
Iustin Pop's avatar
Iustin Pop committed
253 254

  status = child.wait()
255
  return out, err, status
Iustin Pop's avatar
Iustin Pop committed
256

257

258
def _RunCmdFile(cmd, env, via_shell, output, cwd):
259 260 261 262 263 264 265 266 267 268
  """Run a command and save its output to a file.

  @type  cmd: string or list
  @param cmd: Command to run
  @type env: dict
  @param env: The environment to use
  @type via_shell: bool
  @param via_shell: if we should run via the shell
  @type output: str
  @param output: the filename in which to save the output
269 270
  @type cwd: string
  @param cwd: the working directory for the program
271 272 273 274 275 276 277 278 279 280
  @rtype: int
  @return: the exit status

  """
  fh = open(output, "a")
  try:
    child = subprocess.Popen(cmd, shell=via_shell,
                             stderr=subprocess.STDOUT,
                             stdout=fh,
                             stdin=subprocess.PIPE,
281 282
                             close_fds=True, env=env,
                             cwd=cwd)
283 284 285 286 287 288

    child.stdin.close()
    status = child.wait()
  finally:
    fh.close()
  return status
Iustin Pop's avatar
Iustin Pop committed
289 290


291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
def RunParts(dir_name, env=None, reset_env=False):
  """Run Scripts or programs in a directory

  @type dir_name: string
  @param dir_name: absolute path to a directory
  @type env: dict
  @param env: The environment to use
  @type reset_env: boolean
  @param reset_env: whether to reset or keep the default os environment
  @rtype: list of tuples
  @return: list of (name, (one of RUNDIR_STATUS), RunResult)

  """
  rr = []

  try:
    dir_contents = ListVisibleFiles(dir_name)
  except OSError, err:
    logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
    return rr

  for relname in sorted(dir_contents):
    fname = os.path.join(dir_name, relname)
    if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
            constants.EXT_PLUGIN_MASK.match(relname) is not None):
      rr.append((relname, constants.RUNPARTS_SKIP, None))
    else:
      try:
        result = RunCmd([fname], env=env, reset_env=reset_env)
      except Exception, err: # pylint: disable-msg=W0703
        rr.append((relname, constants.RUNPARTS_ERR, str(err)))
      else:
        rr.append((relname, constants.RUNPARTS_RUN, result))

  return rr


Iustin Pop's avatar
Iustin Pop committed
328 329 330 331 332 333
def RemoveFile(filename):
  """Remove a file ignoring some errors.

  Remove a file, ignoring non-existing ones or directories. Other
  errors are passed.

334 335 336
  @type filename: str
  @param filename: the file to be removed

Iustin Pop's avatar
Iustin Pop committed
337 338 339 340
  """
  try:
    os.unlink(filename)
  except OSError, err:
341
    if err.errno not in (errno.ENOENT, errno.EISDIR):
Iustin Pop's avatar
Iustin Pop committed
342 343 344
      raise


345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
  """Renames a file.

  @type old: string
  @param old: Original path
  @type new: string
  @param new: New path
  @type mkdir: bool
  @param mkdir: Whether to create target directory if it doesn't exist
  @type mkdir_mode: int
  @param mkdir_mode: Mode for newly created directories

  """
  try:
    return os.rename(old, new)
  except OSError, err:
    # In at least one use case of this function, the job queue, directory
    # creation is very rare. Checking for the directory before renaming is not
    # as efficient.
    if mkdir and err.errno == errno.ENOENT:
      # Create directory and try again
366 367 368 369 370 371 372 373 374
      dirname = os.path.dirname(new)
      try:
        os.makedirs(dirname, mode=mkdir_mode)
      except OSError, err:
        # Ignore EEXIST. This is only handled in os.makedirs as included in
        # Python 2.5 and above.
        if err.errno != errno.EEXIST or not os.path.exists(dirname):
          raise

375
      return os.rename(old, new)
376

377 378 379
    raise


380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
def ResetTempfileModule():
  """Resets the random name generator of the tempfile module.

  This function should be called after C{os.fork} in the child process to
  ensure it creates a newly seeded random generator. Otherwise it would
  generate the same random parts as the parent process. If several processes
  race for the creation of a temporary file, this could lead to one not getting
  a temporary name.

  """
  # pylint: disable-msg=W0212
  if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
    tempfile._once_lock.acquire()
    try:
      # Reset random name generator
      tempfile._name_sequence = None
    finally:
      tempfile._once_lock.release()
  else:
    logging.critical("The tempfile module misses at least one of the"
                     " '_once_lock' and '_name_sequence' attributes")


Iustin Pop's avatar
Iustin Pop committed
403 404 405 406 407 408
def _FingerprintFile(filename):
  """Compute the fingerprint of a file.

  If the file does not exist, a None will be returned
  instead.

409 410 411 412 413
  @type filename: str
  @param filename: the filename to checksum
  @rtype: str
  @return: the hex digest of the sha checksum of the contents
      of the file
Iustin Pop's avatar
Iustin Pop committed
414 415 416 417 418 419 420

  """
  if not (os.path.exists(filename) and os.path.isfile(filename)):
    return None

  f = open(filename)

421
  fp = sha1()
Iustin Pop's avatar
Iustin Pop committed
422 423 424 425 426 427 428 429 430 431 432 433 434
  while True:
    data = f.read(4096)
    if not data:
      break

    fp.update(data)

  return fp.hexdigest()


def FingerprintFiles(files):
  """Compute fingerprints for a list of files.

435 436 437 438 439
  @type files: list
  @param files: the list of filename to fingerprint
  @rtype: dict
  @return: a dictionary filename: fingerprint, holding only
      existing files
Iustin Pop's avatar
Iustin Pop committed
440 441 442 443 444 445 446 447 448 449 450 451

  """
  ret = {}

  for filename in files:
    cksum = _FingerprintFile(filename)
    if cksum:
      ret[filename] = cksum

  return ret


452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
def ForceDictType(target, key_types, allowed_values=None):
  """Force the values of a dict to have certain types.

  @type target: dict
  @param target: the dict to update
  @type key_types: dict
  @param key_types: dict mapping target dict keys to types
                    in constants.ENFORCEABLE_TYPES
  @type allowed_values: list
  @keyword allowed_values: list of specially allowed values

  """
  if allowed_values is None:
    allowed_values = []

467 468 469 470
  if not isinstance(target, dict):
    msg = "Expected dictionary, got '%s'" % target
    raise errors.TypeEnforcementError(msg)

471 472 473 474 475 476 477 478
  for key in target:
    if key not in key_types:
      msg = "Unknown key '%s'" % key
      raise errors.TypeEnforcementError(msg)

    if target[key] in allowed_values:
      continue

Iustin Pop's avatar
Iustin Pop committed
479 480 481
    ktype = key_types[key]
    if ktype not in constants.ENFORCEABLE_TYPES:
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
482 483
      raise errors.ProgrammerError(msg)

Iustin Pop's avatar
Iustin Pop committed
484
    if ktype == constants.VTYPE_STRING:
485 486 487 488 489 490
      if not isinstance(target[key], basestring):
        if isinstance(target[key], bool) and not target[key]:
          target[key] = ''
        else:
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
          raise errors.TypeEnforcementError(msg)
Iustin Pop's avatar
Iustin Pop committed
491
    elif ktype == constants.VTYPE_BOOL:
492 493 494 495 496 497 498 499 500 501 502 503
      if isinstance(target[key], basestring) and target[key]:
        if target[key].lower() == constants.VALUE_FALSE:
          target[key] = False
        elif target[key].lower() == constants.VALUE_TRUE:
          target[key] = True
        else:
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
          raise errors.TypeEnforcementError(msg)
      elif target[key]:
        target[key] = True
      else:
        target[key] = False
Iustin Pop's avatar
Iustin Pop committed
504
    elif ktype == constants.VTYPE_SIZE:
505 506 507 508 509 510
      try:
        target[key] = ParseUnit(target[key])
      except errors.UnitParseError, err:
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
              (key, target[key], err)
        raise errors.TypeEnforcementError(msg)
Iustin Pop's avatar
Iustin Pop committed
511
    elif ktype == constants.VTYPE_INT:
512 513 514 515 516 517 518
      try:
        target[key] = int(target[key])
      except (ValueError, TypeError):
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
        raise errors.TypeEnforcementError(msg)


Iustin Pop's avatar
Iustin Pop committed
519 520 521
def IsProcessAlive(pid):
  """Check if a given pid exists on the system.

522 523
  @note: zombie status is not handled, so zombie processes
      will be returned as alive
524 525 526 527
  @type pid: int
  @param pid: the process ID to check
  @rtype: boolean
  @return: True if the process exists
Iustin Pop's avatar
Iustin Pop committed
528 529

  """
530 531 532
  if pid <= 0:
    return False

Iustin Pop's avatar
Iustin Pop committed
533
  try:
534 535 536
    os.stat("/proc/%d/status" % pid)
    return True
  except EnvironmentError, err:
537
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
Iustin Pop's avatar
Iustin Pop committed
538
      return False
539
    raise
Iustin Pop's avatar
Iustin Pop committed
540 541


542
def ReadPidFile(pidfile):
543
  """Read a pid from a file.
544

545 546 547
  @type  pidfile: string
  @param pidfile: path to the file containing the pid
  @rtype: int
548
  @return: The process id, if the file exists and contains a valid PID,
549
           otherwise 0
550 551 552

  """
  try:
553
    raw_data = ReadFile(pidfile)
554 555
  except EnvironmentError, err:
    if err.errno != errno.ENOENT:
556
      logging.exception("Can't read pid file")
557
    return 0
558 559

  try:
560
    pid = int(raw_data)
561
  except (TypeError, ValueError), err:
562
    logging.info("Can't parse pid file contents", exc_info=True)
563
    return 0
564

565
  return pid
566 567


568
def MatchNameComponent(key, name_list, case_sensitive=True):
Iustin Pop's avatar
Iustin Pop committed
569 570 571
  """Try to match a name against a list.

  This function will try to match a name like test1 against a list
572 573 574 575
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
  this list, I{'test1'} as well as I{'test1.example'} will match, but
  not I{'test1.ex'}. A multiple match will be considered as no match
  at all (e.g. I{'test1'} against C{['test1.example.com',
576 577
  'test1.example.org']}), except when the key fully matches an entry
  (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
Iustin Pop's avatar
Iustin Pop committed
578

579 580 581 582
  @type key: str
  @param key: the name to be searched
  @type name_list: list
  @param name_list: the list of strings against which to search the key
583 584
  @type case_sensitive: boolean
  @param case_sensitive: whether to provide a case-sensitive match
Iustin Pop's avatar
Iustin Pop committed
585

586 587 588
  @rtype: None or str
  @return: None if there is no match I{or} if there are multiple matches,
      otherwise the element from the list which matches
Iustin Pop's avatar
Iustin Pop committed
589 590

  """
591 592
  if key in name_list:
    return key
593 594 595 596

  re_flags = 0
  if not case_sensitive:
    re_flags |= re.IGNORECASE
597
    key = key.upper()
598 599 600 601 602 603
  mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
  names_filtered = []
  string_matches = []
  for name in name_list:
    if mo.match(name) is not None:
      names_filtered.append(name)
604
      if not case_sensitive and key == name.upper():
605 606 607 608 609 610 611
        string_matches.append(name)

  if len(string_matches) == 1:
    return string_matches[0]
  if len(names_filtered) == 1:
    return names_filtered[0]
  return None
Iustin Pop's avatar
Iustin Pop committed
612 613


614
class HostInfo:
615
  """Class implementing resolver and hostname functionality
616 617

  """
618
  def __init__(self, name=None):
619 620
    """Initialize the host name object.

621 622
    If the name argument is not passed, it will use this system's
    name.
623 624

    """
625 626 627 628 629
    if name is None:
      name = self.SysName()

    self.query = name
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
630 631
    self.ip = self.ipaddrs[0]

632 633 634 635 636 637
  def ShortName(self):
    """Returns the hostname without domain.

    """
    return self.name.split('.')[0]

638 639 640
  @staticmethod
  def SysName():
    """Return the current system's name.
641

642
    This is simply a wrapper over C{socket.gethostname()}.
Iustin Pop's avatar
Iustin Pop committed
643

644 645
    """
    return socket.gethostname()
Iustin Pop's avatar
Iustin Pop committed
646

647 648 649
  @staticmethod
  def LookupHostname(hostname):
    """Look up hostname
Iustin Pop's avatar
Iustin Pop committed
650

651 652
    @type hostname: str
    @param hostname: hostname to look up
653

654 655 656 657
    @rtype: tuple
    @return: a tuple (name, aliases, ipaddrs) as returned by
        C{socket.gethostbyname_ex}
    @raise errors.ResolverError: in case of errors in resolving
658 659 660 661 662 663 664

    """
    try:
      result = socket.gethostbyname_ex(hostname)
    except socket.gaierror, err:
      # hostname not found in DNS
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
Iustin Pop's avatar
Iustin Pop committed
665

666
    return result
Iustin Pop's avatar
Iustin Pop committed
667 668


669 670 671 672 673 674 675 676 677 678
def GetHostInfo(name=None):
  """Lookup host name and raise an OpPrereqError for failures"""

  try:
    return HostInfo(name)
  except errors.ResolverError, err:
    raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
                               (err[0], err[2]), errors.ECODE_RESOLVER)


Iustin Pop's avatar
Iustin Pop committed
679 680 681
def ListVolumeGroups():
  """List volume groups and their size

682 683 684 685
  @rtype: dict
  @return:
       Dictionary with keys volume name and values
       the size of the volume
Iustin Pop's avatar
Iustin Pop committed
686 687 688 689 690 691 692 693 694 695 696 697 698

  """
  command = "vgs --noheadings --units m --nosuffix -o name,size"
  result = RunCmd(command)
  retval = {}
  if result.failed:
    return retval

  for line in result.stdout.splitlines():
    try:
      name, size = line.split()
      size = int(float(size))
    except (IndexError, ValueError), err:
699
      logging.error("Invalid output from vgs (%s): %s", err, line)
Iustin Pop's avatar
Iustin Pop committed
700 701 702 703 704 705 706 707 708 709
      continue

    retval[name] = size

  return retval


def BridgeExists(bridge):
  """Check whether the given bridge exists in the system

710 711 712 713
  @type bridge: str
  @param bridge: the bridge name to check
  @rtype: boolean
  @return: True if it does
Iustin Pop's avatar
Iustin Pop committed
714 715 716 717 718 719 720 721

  """
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)


def NiceSort(name_list):
  """Sort a list of strings based on digit and non-digit groupings.

722 723 724
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
  will sort the list in the logical order C{['a1', 'a2', 'a10',
  'a11']}.
Iustin Pop's avatar
Iustin Pop committed
725 726 727 728 729

  The sort algorithm breaks each name in groups of either only-digits
  or no-digits. Only the first eight such groups are considered, and
  after that we just use what's left of the string.

730 731 732 733
  @type name_list: list
  @param name_list: the names to be sorted
  @rtype: list
  @return: a copy of the name list sorted with our algorithm
Iustin Pop's avatar
Iustin Pop committed
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758

  """
  _SORTER_BASE = "(\D+|\d+)"
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE,
                                                  _SORTER_BASE, _SORTER_BASE)
  _SORTER_RE = re.compile(_SORTER_FULL)
  _SORTER_NODIGIT = re.compile("^\D*$")
  def _TryInt(val):
    """Attempts to convert a variable to integer."""
    if val is None or _SORTER_NODIGIT.match(val):
      return val
    rval = int(val)
    return rval

  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
             for name in name_list]
  to_sort.sort()
  return [tup[1] for tup in to_sort]


def TryConvert(fn, val):
  """Try to convert a value ignoring errors.

759 760 761 762 763 764 765 766 767 768
  This function tries to apply function I{fn} to I{val}. If no
  C{ValueError} or C{TypeError} exceptions are raised, it will return
  the result, else it will return the original value. Any other
  exceptions are propagated to the caller.

  @type fn: callable
  @param fn: function to apply to the value
  @param val: the value to be converted
  @return: The converted value if the conversion was successful,
      otherwise the original value.
Iustin Pop's avatar
Iustin Pop committed
769 770 771 772

  """
  try:
    nv = fn(val)
Michael Hanselmann's avatar
Michael Hanselmann committed
773
  except (ValueError, TypeError):
Iustin Pop's avatar
Iustin Pop committed
774 775 776 777 778
    nv = val
  return nv


def IsValidIP(ip):
779
  """Verifies the syntax of an IPv4 address.
Iustin Pop's avatar
Iustin Pop committed
780

781 782 783 784 785 786
  This function checks if the IPv4 address passes is valid or not based
  on syntax (not IP range, class calculations, etc.).

  @type ip: str
  @param ip: the address to be checked
  @rtype: a regular expression match object
Michael Hanselmann's avatar
Michael Hanselmann committed
787
  @return: a regular expression match object, or None if the
788
      address is not valid
Iustin Pop's avatar
Iustin Pop committed
789 790 791

  """
  unit = "(0|[1-9]\d{0,2})"
792
  #TODO: convert and return only boolean
Iustin Pop's avatar
Iustin Pop committed
793 794 795 796 797 798 799 800 801 802 803 804 805
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)


def IsValidShellParam(word):
  """Verifies is the given word is safe from the shell's p.o.v.

  This means that we can pass this to a command via the shell and be
  sure that it doesn't alter the command line and is passed as such to
  the actual command.

  Note that we are overly restrictive here, in order to be on the safe
  side.

806 807 808 809 810
  @type word: str
  @param word: the word to check
  @rtype: boolean
  @return: True if the word is 'safe'

Iustin Pop's avatar
Iustin Pop committed
811 812 813 814 815 816 817 818 819
  """
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))


def BuildShellCmd(template, *args):
  """Build a safe shell command line from the given arguments.

  This function will check all arguments in the args list so that they
  are valid shell parameters (i.e. they don't contain shell
Michael Hanselmann's avatar
Michael Hanselmann committed
820
  metacharacters). If everything is ok, it will return the result of
Iustin Pop's avatar
Iustin Pop committed
821 822
  template % args.

823 824 825 826 827 828
  @type template: str
  @param template: the string holding the template for the
      string formatting
  @rtype: str
  @return: the expanded command line

Iustin Pop's avatar
Iustin Pop committed
829 830 831
  """
  for word in args:
    if not IsValidShellParam(word):
832 833
      raise errors.ProgrammerError("Shell argument '%s' contains"
                                   " invalid characters" % word)
Iustin Pop's avatar
Iustin Pop committed
834 835 836
  return template % args


837
def FormatUnit(value, units):
Iustin Pop's avatar
Iustin Pop committed
838 839
  """Formats an incoming number of MiB with the appropriate unit.

840 841
  @type value: int
  @param value: integer representing the value in MiB (1048576)
842 843 844 845 846 847
  @type units: char
  @param units: the type of formatting we should do:
      - 'h' for automatic scaling
      - 'm' for MiBs
      - 'g' for GiBs
      - 't' for TiBs
848 849
  @rtype: str
  @return: the formatted value (with suffix)
Iustin Pop's avatar
Iustin Pop committed
850 851

  """
852 853
  if units not in ('m', 'g', 't', 'h'):
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
Iustin Pop's avatar
Iustin Pop committed
854

855 856 857 858 859 860 861 862 863 864 865
  suffix = ''

  if units == 'm' or (units == 'h' and value < 1024):
    if units == 'h':
      suffix = 'M'
    return "%d%s" % (round(value, 0), suffix)

  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
    if units == 'h':
      suffix = 'G'
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
Iustin Pop's avatar
Iustin Pop committed
866 867

  else:
868 869 870
    if units == 'h':
      suffix = 'T'
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
Iustin Pop's avatar
Iustin Pop committed
871 872 873 874 875


def ParseUnit(input_string):
  """Tries to extract number and scale from the given string.

876 877 878
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
  is always an int in MiB.
Iustin Pop's avatar
Iustin Pop committed
879 880

  """
881
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
Iustin Pop's avatar
Iustin Pop committed
882
  if not m:
883
    raise errors.UnitParseError("Invalid format")
Iustin Pop's avatar
Iustin Pop committed
884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903

  value = float(m.groups()[0])

  unit = m.groups()[1]
  if unit:
    lcunit = unit.lower()
  else:
    lcunit = 'm'

  if lcunit in ('m', 'mb', 'mib'):
    # Value already in MiB
    pass

  elif lcunit in ('g', 'gb', 'gib'):
    value *= 1024

  elif lcunit in ('t', 'tb', 'tib'):
    value *= 1024 * 1024

  else:
904
    raise errors.UnitParseError("Unknown unit: %s" % unit)
Iustin Pop's avatar
Iustin Pop committed
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920

  # Make sure we round up
  if int(value) < value:
    value += 1

  # Round up to the next multiple of 4
  value = int(value)
  if value % 4:
    value += 4 - value % 4

  return value


def AddAuthorizedKey(file_name, key):
  """Adds an SSH public key to an authorized_keys file.

921 922 923 924 925
  @type file_name: str
  @param file_name: path to authorized_keys file
  @type key: str
  @param key: string containing key

Iustin Pop's avatar
Iustin Pop committed
926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
  """
  key_fields = key.split()

  f = open(file_name, 'a+')
  try:
    nl = True
    for line in f:
      # Ignore whitespace changes
      if line.split() == key_fields:
        break
      nl = line.endswith('\n')
    else:
      if not nl:
        f.write("\n")
      f.write(key.rstrip('\r\n'))
      f.write("\n")
      f.flush()
  finally:
    f.close()


def RemoveAuthorizedKey(file_name, key):
  """Removes an SSH public key from an authorized_keys file.

950 951 952 953 954
  @type file_name: str
  @param file_name: path to authorized_keys file
  @type key: str
  @param key: string containing key

Iustin Pop's avatar
Iustin Pop committed
955 956 957 958 959
  """
  key_fields = key.split()

  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
960
    out = os.fdopen(fd, 'w')
Iustin Pop's avatar
Iustin Pop committed
961
    try:
962 963 964 965 966 967
      f = open(file_name, 'r')
      try:
        for line in f:
          # Ignore whitespace changes while comparing lines
          if line.split() != key_fields:
            out.write(line)
968 969 970 971 972 973 974 975 976 977 978 979

        out.flush()
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise


980 981
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
  """Sets the name of an IP address and hostname in /etc/hosts.
982

983 984 985 986 987 988 989 990 991
  @type file_name: str
  @param file_name: path to the file to modify (usually C{/etc/hosts})
  @type ip: str
  @param ip: the IP address
  @type hostname: str
  @param hostname: the hostname to be added
  @type aliases: list
  @param aliases: the list of aliases to add for the hostname

992
  """
993
  # FIXME: use WriteFile + fn rather than duplicating its efforts
994 995 996
  # Ensure aliases are unique
  aliases = UniqueSequence([hostname] + aliases)[1:]

997
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
998
  try:
999 1000 1001 1002 1003 1004
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        for line in f:
          fields = line.split()
1005
          if fields and not fields[0].startswith('#') and ip == fields[0]:
1006 1007 1008
            continue
          out.write(line)

1009
        out.write("%s\t%s" % (ip, hostname))
1010 1011 1012 1013 1014
        if aliases:
          out.write(" %s" % ' '.join(aliases))
        out.write('\n')

        out.flush()
1015
        os.fsync(out)
1016
        os.chmod(tmpname, 0644)
1017 1018 1019 1020 1021 1022 1023 1024
        os.rename(tmpname, file_name)
      finally:
        f.close()
    finally:
      out.close()
  except:
    RemoveFile(tmpname)
    raise
1025 1026


1027 1028 1029
def AddHostToEtcHosts(hostname):
  """Wrapper around SetEtcHostsEntry.

1030 1031 1032 1033
  @type hostname: str
  @param hostname: a hostname that will be resolved and added to
      L{constants.ETC_HOSTS}

1034 1035 1036 1037 1038
  """
  hi = HostInfo(name=hostname)
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])


1039
def RemoveEtcHostsEntry(file_name, hostname):
1040
  """Removes a hostname from /etc/hosts.
1041

1042
  IP addresses without names are removed from the file.
1043 1044 1045 1046 1047 1048

  @type file_name: str
  @param file_name: path to the file to modify (usually C{/etc/hosts})
  @type hostname: str
  @param hostname: the hostname to be removed

1049
  """
1050
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
  try:
    out = os.fdopen(fd, 'w')
    try:
      f = open(file_name, 'r')
      try:
        for line in f:
          fields = line.split()
          if len(fields) > 1 and not fields[0].startswith('#'):
            names = fields[1:]
            if hostname in names:
              while hostname in names:
                names.remove(hostname)
              if names:
1065
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
1066 1067 1068
              continue

          out.write(line)
1069 1070

        out.flush()
1071
        os.fsync(out)
1072
        os.chmod(tmpname, 0644)
1073 1074 1075
        os.rename(tmpname, file_name)
      finally:
        f.close()
Iustin Pop's avatar
Iustin Pop committed
1076
    finally:
1077 1078 1079 1080
      out.close()
  except:
    RemoveFile(tmpname)
    raise
Iustin Pop's avatar
Iustin Pop committed
1081 1082


1083 1084 1085
def RemoveHostFromEtcHosts(hostname):
  """Wrapper around RemoveEtcHostsEntry.

1086 1087 1088 1089 1090
  @type hostname: str
  @param hostname: hostname that will be resolved and its
      full and shot name will be removed from
      L{constants.ETC_HOSTS}

1091 1092 1093 1094 1095 1096
  """
  hi = HostInfo(name=hostname)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())


Iustin Pop's avatar
Iustin Pop committed
1097 1098 1099
def CreateBackup(file_name):
  """Creates a backup of a file.

1100 1101 1102 1103 1104
  @type file_name: str
  @param file_name: file to be backed up
  @rtype: str
  @return: the path to the newly created backup
  @raise errors.ProgrammerError: for invalid file names
Iustin Pop's avatar
Iustin Pop committed
1105 1106 1107

  """
  if not os.path.isfile(file_name):
1108 1109
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
                                file_name)
Iustin Pop's avatar
Iustin Pop committed
1110

1111
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
Iustin Pop's avatar
Iustin Pop committed
1112
  dir_name = os.path.dirname(file_name)
1113 1114 1115

  fsrc = open(file_name, 'rb')
  try:
Iustin Pop's avatar
Iustin Pop committed
1116
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1117 1118 1119 1120 1121 1122 1123 1124
    fdst = os.fdopen(fd, 'wb')
    try:
      shutil.copyfileobj(fsrc, fdst)
    finally:
      fdst.close()
  finally:
    fsrc.close()

Iustin Pop's avatar
Iustin Pop committed
1125 1126 1127 1128 1129
  return backup_name


def ShellQuote(value):
  """Quotes shell argument according to POSIX.
1130

1131 1132 1133 1134 1135
  @type value: str
  @param value: the argument to be quoted
  @rtype: str
  @return: the quoted value

Iustin Pop's avatar
Iustin Pop committed
1136 1137 1138 1139 1140 1141 1142 1143
  """
  if _re_shell_unquoted.match(value):
    return value
  else:
    return "'%s'" % value.replace("'", "'\\''")


def ShellQuoteArgs(args):
1144 1145 1146 1147 1148
  """Quotes a list of shell arguments.

  @type args: list
  @param args: list of arguments to be quoted
  @rtype: str
Michael Hanselmann's avatar
Michael Hanselmann committed
1149
  @return: the quoted arguments concatenated with spaces
Iustin Pop's avatar
Iustin Pop committed
1150 1151 1152

  """
  return ' '.join([ShellQuote(i) for i in args])
1153 1154


1155
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1156 1157
  """Simple ping implementation using TCP connect(2).

1158 1159 1160 1161 1162 1163 1164 1165
  Check if the given IP is reachable by doing attempting a TCP connect
  to it.

  @type target: str
  @param target: the IP or hostname to ping
  @type port: int
  @param port: the port to connect to
  @type timeout: int
Michael Hanselmann's avatar
Michael Hanselmann committed
1166
  @param timeout: the timeout on the connection attempt
1167 1168 1169 1170 1171 1172 1173
  @type live_port_needed: boolean
  @param live_port_needed: whether a closed port will cause the
      function to return failure, as if there was a timeout
  @type source: str or None
  @param source: if specified, will cause the connect to be made
      from this specific source address; failures to bind other
      than C{EADDRNOTAVAIL} will be ignored
1174 1175 1176 1177

  """
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)