cli.py

#
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Module dealing with command line parsing"""


import sys
import textwrap
import os.path
import time
import logging
from cStringIO import StringIO

from ganeti import utils
from ganeti import errors
from ganeti import constants
from ganeti import opcodes
from ganeti import luxi
from ganeti import ssconf
from ganeti import rpc
from ganeti import ssh

from optparse import (OptionParser, TitledHelpFormatter,
                      Option, OptionValueError)


__all__ = [
  # Command line options
  "ALLOCATABLE_OPT",
  "ALL_OPT",
  "AUTO_PROMOTE_OPT",
  "AUTO_REPLACE_OPT",
  "BACKEND_OPT",
  "CLEANUP_OPT",
  "CONFIRM_OPT",
  "CP_SIZE_OPT",
  "DEBUG_OPT",
  "DEBUG_SIMERR_OPT",
  "DISKIDX_OPT",
  "DISK_OPT",
  "DISK_TEMPLATE_OPT",
  "DRAINED_OPT",
  "EARLY_RELEASE_OPT",
  "ENABLED_HV_OPT",
  "ERROR_CODES_OPT",
  "FIELDS_OPT",
  "FILESTORE_DIR_OPT",
  "FILESTORE_DRIVER_OPT",
  "FORCE_OPT",
  "FORCE_VARIANT_OPT",
  "GLOBAL_FILEDIR_OPT",
  "HVLIST_OPT",
  "HVOPTS_OPT",
  "HYPERVISOR_OPT",
  "IALLOCATOR_OPT",
  "IGNORE_CONSIST_OPT",
  "IGNORE_FAILURES_OPT",
  "IGNORE_SECONDARIES_OPT",
  "IGNORE_SIZE_OPT",
  "MAC_PREFIX_OPT",
  "MASTER_NETDEV_OPT",
  "MC_OPT",
  "NET_OPT",
  "NEW_CLUSTER_CERT_OPT",
  "NEW_CONFD_HMAC_KEY_OPT",
  "NEW_RAPI_CERT_OPT",
  "NEW_SECONDARY_OPT",
  "NIC_PARAMS_OPT",
  "NODE_LIST_OPT",
  "NODE_PLACEMENT_OPT",
  "NOHDR_OPT",
  "NOIPCHECK_OPT",
  "NO_INSTALL_OPT",
  "NONAMECHECK_OPT",
  "NOLVM_STORAGE_OPT",
  "NOMODIFY_ETCHOSTS_OPT",
  "NOMODIFY_SSH_SETUP_OPT",
  "NONICS_OPT",
  "NONLIVE_OPT",
  "NONPLUS1_OPT",
  "NOSHUTDOWN_OPT",
  "NOSTART_OPT",
  "NOSSH_KEYCHECK_OPT",
  "NOVOTING_OPT",
  "NWSYNC_OPT",
  "ON_PRIMARY_OPT",
  "ON_SECONDARY_OPT",
  "OFFLINE_OPT",
  "OS_OPT",
  "OS_SIZE_OPT",
  "RAPI_CERT_OPT",
  "READD_OPT",
  "REBOOT_TYPE_OPT",
  "SECONDARY_IP_OPT",
  "SELECT_OS_OPT",
  "SEP_OPT",
  "SHOWCMD_OPT",
  "SHUTDOWN_TIMEOUT_OPT",
  "SINGLE_NODE_OPT",
  "SRC_DIR_OPT",
  "SRC_NODE_OPT",
  "SUBMIT_OPT",
  "STATIC_OPT",
  "SYNC_OPT",
  "TAG_SRC_OPT",
  "TIMEOUT_OPT",
  "USEUNITS_OPT",
  "VERBOSE_OPT",
  "VG_NAME_OPT",
  "YES_DOIT_OPT",
  # Generic functions for CLI programs
  "GenericMain",
  "GenericInstanceCreate",
  "GetClient",
  "GetOnlineNodes",
  "JobExecutor",
  "JobSubmittedException",
  "ParseTimespec",
  "RunWhileClusterStopped",
  "SubmitOpCode",
  "SubmitOrSend",
  "UsesRPC",
  # Formatting functions
  "ToStderr", "ToStdout",
  "FormatError",
  "GenerateTable",
  "AskUser",
  "FormatTimestamp",
  # Tags functions
  "ListTags",
  "AddTags",
  "RemoveTags",
  # command line options support infrastructure
  "ARGS_MANY_INSTANCES",
  "ARGS_MANY_NODES",
  "ARGS_NONE",
  "ARGS_ONE_INSTANCE",
  "ARGS_ONE_NODE",
  "ARGS_ONE_OS",
  "ArgChoice",
  "ArgCommand",
  "ArgFile",
  "ArgHost",
  "ArgInstance",
  "ArgJobId",
  "ArgNode",
  "ArgOs",
  "ArgSuggest",
  "ArgUnknown",
  "OPT_COMPL_INST_ADD_NODES",
  "OPT_COMPL_MANY_NODES",
  "OPT_COMPL_ONE_IALLOCATOR",
  "OPT_COMPL_ONE_INSTANCE",
  "OPT_COMPL_ONE_NODE",
  "OPT_COMPL_ONE_OS",
  "cli_option",
  "SplitNodeOption",
  "CalculateOSNames",
  ]

NO_PREFIX = "no_"
UN_PREFIX = "-"


class _Argument:
  def __init__(self, min=0, max=None): # pylint: disable-msg=W0622
    self.min = min
    self.max = max

  def __repr__(self):
    return ("<%s min=%s max=%s>" %
            (self.__class__.__name__, self.min, self.max))


class ArgSuggest(_Argument):
  """Suggesting argument.

  Value can be any of the ones passed to the constructor.

  """
  # pylint: disable-msg=W0622
  def __init__(self, min=0, max=None, choices=None):
    _Argument.__init__(self, min=min, max=max)
    self.choices = choices

  def __repr__(self):
    return ("<%s min=%s max=%s choices=%r>" %
            (self.__class__.__name__, self.min, self.max, self.choices))


class ArgChoice(ArgSuggest):
  """Choice argument.

  Value can be any of the ones passed to the constructor. Like L{ArgSuggest},
  but value must be one of the choices.

  """


class ArgUnknown(_Argument):
  """Unknown argument to program (e.g. determined at runtime).

  """


class ArgInstance(_Argument):
  """Instances argument.

  """


class ArgNode(_Argument):
  """Node argument.

  """

class ArgJobId(_Argument):
  """Job ID argument.

  """


class ArgFile(_Argument):
  """File path argument.

  """


class ArgCommand(_Argument):
  """Command argument.

  """


class ArgHost(_Argument):
  """Host argument.

  """


class ArgOs(_Argument):
  """OS argument.

  """


ARGS_NONE = []
ARGS_MANY_INSTANCES = [ArgInstance()]
ARGS_MANY_NODES = [ArgNode()]
ARGS_ONE_INSTANCE = [ArgInstance(min=1, max=1)]
ARGS_ONE_NODE = [ArgNode(min=1, max=1)]
ARGS_ONE_OS = [ArgOs(min=1, max=1)]


def _ExtractTagsObject(opts, args):
  """Extract the tag type object.

  Note that this function will modify its args parameter.

  """
  if not hasattr(opts, "tag_type"):
    raise errors.ProgrammerError("tag_type not passed to _ExtractTagsObject")
  kind = opts.tag_type
  if kind == constants.TAG_CLUSTER:
    retval = kind, kind
  elif kind == constants.TAG_NODE or kind == constants.TAG_INSTANCE:
    if not args:
      raise errors.OpPrereqError("no arguments passed to the command")
    name = args.pop(0)
    retval = kind, name
  else:
    raise errors.ProgrammerError("Unhandled tag type '%s'" % kind)
  return retval


def _ExtendTags(opts, args):
  """Extend the args if a source file has been given.

  This function will extend the tags with the contents of the file
  passed in the 'tags_source' attribute of the opts parameter. A file
  named '-' will be replaced by stdin.

  """
  fname = opts.tags_source
  if fname is None:
    return
  if fname == "-":
    new_fh = sys.stdin
  else:
    new_fh = open(fname, "r")
  new_data = []
  try:
    # we don't use the nice 'new_data = [line.strip() for line in fh]'
    # because of python bug 1633941
    while True:
      line = new_fh.readline()
      if not line:
        break
      new_data.append(line.strip())
  finally:
    new_fh.close()
  args.extend(new_data)


def ListTags(opts, args):
  """List the tags on a given object.

  This is a generic implementation that knows how to deal with all
  three cases of tag objects (cluster, node, instance). The opts
  argument is expected to contain a tag_type field denoting what
  object type we work on.

  """
  kind, name = _ExtractTagsObject(opts, args)
  cl = GetClient()
  result = cl.QueryTags(kind, name)
  result = list(result)
  result.sort()
  for tag in result:
    ToStdout(tag)


def AddTags(opts, args):
  """Add tags on a given object.

  This is a generic implementation that knows how to deal with all
  three cases of tag objects (cluster, node, instance). The opts
  argument is expected to contain a tag_type field denoting what
  object type we work on.

  """
  kind, name = _ExtractTagsObject(opts, args)
  _ExtendTags(opts, args)
  if not args:
    raise errors.OpPrereqError("No tags to be added")
  op = opcodes.OpAddTags(kind=kind, name=name, tags=args)
  SubmitOpCode(op)


def RemoveTags(opts, args):
  """Remove tags from a given object.

  This is a generic implementation that knows how to deal with all
  three cases of tag objects (cluster, node, instance). The opts
  argument is expected to contain a tag_type field denoting what
  object type we work on.

  """
  kind, name = _ExtractTagsObject(opts, args)
  _ExtendTags(opts, args)
  if not args:
    raise errors.OpPrereqError("No tags to be removed")
  op = opcodes.OpDelTags(kind=kind, name=name, tags=args)
  SubmitOpCode(op)


def check_unit(option, opt, value): # pylint: disable-msg=W0613
  """OptParsers custom converter for units.

  """
  try:
    return utils.ParseUnit(value)
  except errors.UnitParseError, err:
    raise OptionValueError("option %s: %s" % (opt, err))


def _SplitKeyVal(opt, data):
  """Convert a KeyVal string into a dict.

  This function will convert a key=val[,...] string into a dict. Empty
  values will be converted specially: keys which have the prefix 'no_'
  will have the value=False and the prefix stripped, the others will
  have value=True.

  @type opt: string
  @param opt: a string holding the option name for which we process the
      data, used in building error messages
  @type data: string
  @param data: a string of the format key=val,key=val,...
  @rtype: dict
  @return: {key=val, key=val}
  @raises errors.ParameterError: if there are duplicate keys

  """
  kv_dict = {}
  if data:
    for elem in utils.UnescapeAndSplit(data, sep=","):
      if "=" in elem:
        key, val = elem.split("=", 1)
      else:
        if elem.startswith(NO_PREFIX):
          key, val = elem[len(NO_PREFIX):], False
        elif elem.startswith(UN_PREFIX):
          key, val = elem[len(UN_PREFIX):], None
        else:
          key, val = elem, True
      if key in kv_dict:
        raise errors.ParameterError("Duplicate key '%s' in option %s" %
                                    (key, opt))
      kv_dict[key] = val
  return kv_dict


def check_ident_key_val(option, opt, value):  # pylint: disable-msg=W0613
  """Custom parser for ident:key=val,key=val options.

  This will store the parsed values as a tuple (ident, {key: val}). As such,
  multiple uses of this option via action=append is possible.

  """
  if ":" not in value:
    ident, rest = value, ''
  else:
    ident, rest = value.split(":", 1)

  if ident.startswith(NO_PREFIX):
    if rest:
      msg = "Cannot pass options when removing parameter groups: %s" % value
      raise errors.ParameterError(msg)
    retval = (ident[len(NO_PREFIX):], False)
  elif ident.startswith(UN_PREFIX):
    if rest:
      msg = "Cannot pass options when removing parameter groups: %s" % value
      raise errors.ParameterError(msg)
    retval = (ident[len(UN_PREFIX):], None)
  else:
    kv_dict = _SplitKeyVal(opt, rest)
    retval = (ident, kv_dict)
  return retval


def check_key_val(option, opt, value):  # pylint: disable-msg=W0613
  """Custom parser class for key=val,key=val options.

  This will store the parsed values as a dict {key: val}.

  """
  return _SplitKeyVal(opt, value)


# completion_suggestion is normally a list. Using numeric values not evaluating
# to False for dynamic completion.
(OPT_COMPL_MANY_NODES,
 OPT_COMPL_ONE_NODE,
 OPT_COMPL_ONE_INSTANCE,
 OPT_COMPL_ONE_OS,
 OPT_COMPL_ONE_IALLOCATOR,
 OPT_COMPL_INST_ADD_NODES) = range(100, 106)

OPT_COMPL_ALL = frozenset([
  OPT_COMPL_MANY_NODES,
  OPT_COMPL_ONE_NODE,
  OPT_COMPL_ONE_INSTANCE,
  OPT_COMPL_ONE_OS,
  OPT_COMPL_ONE_IALLOCATOR,
  OPT_COMPL_INST_ADD_NODES,
  ])


class CliOption(Option):
  """Custom option class for optparse.

  """
  ATTRS = Option.ATTRS + [
    "completion_suggest",
    ]
  TYPES = Option.TYPES + (
    "identkeyval",
    "keyval",
    "unit",
    )
  TYPE_CHECKER = Option.TYPE_CHECKER.copy()
  TYPE_CHECKER["identkeyval"] = check_ident_key_val
  TYPE_CHECKER["keyval"] = check_key_val
  TYPE_CHECKER["unit"] = check_unit


# optparse.py sets make_option, so we do it for our own option class, too
cli_option = CliOption


_YESNO = ("yes", "no")
_YORNO = "yes|no"

DEBUG_OPT = cli_option("-d", "--debug", default=0, action="count",
                       help="Increase debugging level")

NOHDR_OPT = cli_option("--no-headers", default=False,
                       action="store_true", dest="no_headers",
                       help="Don't display column headers")

SEP_OPT = cli_option("--separator", default=None,
                     action="store", dest="separator",
                     help=("Separator between output fields"
                           " (defaults to one space)"))

USEUNITS_OPT = cli_option("--units", default=None,
                          dest="units", choices=('h', 'm', 'g', 't'),
                          help="Specify units for output (one of hmgt)")

FIELDS_OPT = cli_option("-o", "--output", dest="output", action="store",
                        type="string", metavar="FIELDS",
                        help="Comma separated list of output fields")

FORCE_OPT = cli_option("-f", "--force", dest="force", action="store_true",
                       default=False, help="Force the operation")

CONFIRM_OPT = cli_option("--yes", dest="confirm", action="store_true",
                         default=False, help="Do not require confirmation")

TAG_SRC_OPT = cli_option("--from", dest="tags_source",
                         default=None, help="File with tag names")

SUBMIT_OPT = cli_option("--submit", dest="submit_only",
                        default=False, action="store_true",
                        help=("Submit the job and return the job ID, but"
                              " don't wait for the job to finish"))

SYNC_OPT = cli_option("--sync", dest="do_locking",
                      default=False, action="store_true",
                      help=("Grab locks while doing the queries"
                            " in order to ensure more consistent results"))

_DRY_RUN_OPT = cli_option("--dry-run", default=False,
                          action="store_true",
                          help=("Do not execute the operation, just run the"
                                " check steps and verify it it could be"
                                " executed"))

VERBOSE_OPT = cli_option("-v", "--verbose", default=False,
                         action="store_true",
                         help="Increase the verbosity of the operation")

DEBUG_SIMERR_OPT = cli_option("--debug-simulate-errors", default=False,
                              action="store_true", dest="simulate_errors",
                              help="Debugging option that makes the operation"
                              " treat most runtime checks as failed")

NWSYNC_OPT = cli_option("--no-wait-for-sync", dest="wait_for_sync",
                        default=True, action="store_false",
                        help="Don't wait for sync (DANGEROUS!)")

DISK_TEMPLATE_OPT = cli_option("-t", "--disk-template", dest="disk_template",
                               help="Custom disk setup (diskless, file,"
                               " plain or drbd)",
                               default=None, metavar="TEMPL",
                               choices=list(constants.DISK_TEMPLATES))

NONICS_OPT = cli_option("--no-nics", default=False, action="store_true",
                        help="Do not create any network cards for"
                        " the instance")

FILESTORE_DIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir",
                               help="Relative path under default cluster-wide"
                               " file storage dir to store file-based disks",
                               default=None, metavar="<DIR>")

FILESTORE_DRIVER_OPT = cli_option("--file-driver", dest="file_driver",
                                  help="Driver to use for image files",
                                  default="loop", metavar="<DRIVER>",
                                  choices=list(constants.FILE_DRIVER))

IALLOCATOR_OPT = cli_option("-I", "--iallocator", metavar="<NAME>",
                            help="Select nodes for the instance automatically"
                            " using the <NAME> iallocator plugin",
                            default=None, type="string",
                            completion_suggest=OPT_COMPL_ONE_IALLOCATOR)

OS_OPT = cli_option("-o", "--os-type", dest="os", help="What OS to run",
                    metavar="<os>",
                    completion_suggest=OPT_COMPL_ONE_OS)

FORCE_VARIANT_OPT = cli_option("--force-variant", dest="force_variant",
                               action="store_true", default=False,
                               help="Force an unknown variant")

NO_INSTALL_OPT = cli_option("--no-install", dest="no_install",
                            action="store_true", default=False,
                            help="Do not install the OS (will"
                            " enable no-start)")

BACKEND_OPT = cli_option("-B", "--backend-parameters", dest="beparams",
                         type="keyval", default={},
                         help="Backend parameters")

HVOPTS_OPT =  cli_option("-H", "--hypervisor-parameters", type="keyval",
                         default={}, dest="hvparams",
                         help="Hypervisor parameters")

HYPERVISOR_OPT = cli_option("-H", "--hypervisor-parameters", dest="hypervisor",
                            help="Hypervisor and hypervisor options, in the"
                            " format hypervisor:option=value,option=value,...",
                            default=None, type="identkeyval")

HVLIST_OPT = cli_option("-H", "--hypervisor-parameters", dest="hvparams",
                        help="Hypervisor and hypervisor options, in the"
                        " format hypervisor:option=value,option=value,...",
                        default=[], action="append", type="identkeyval")

NOIPCHECK_OPT = cli_option("--no-ip-check", dest="ip_check", default=True,
                           action="store_false",
                           help="Don't check that the instance's IP"
                           " is alive")

NONAMECHECK_OPT = cli_option("--no-name-check", dest="name_check",
                             default=True, action="store_false",
                             help="Don't check that the instance's name"
                             " is resolvable")

NET_OPT = cli_option("--net",
                     help="NIC parameters", default=[],
                     dest="nics", action="append", type="identkeyval")

DISK_OPT = cli_option("--disk", help="Disk parameters", default=[],
                      dest="disks", action="append", type="identkeyval")

DISKIDX_OPT = cli_option("--disks", dest="disks", default=None,
                         help="Comma-separated list of disks"
                         " indices to act on (e.g. 0,2) (optional,"
                         " defaults to all disks)")

OS_SIZE_OPT = cli_option("-s", "--os-size", dest="sd_size",
                         help="Enforces a single-disk configuration using the"
                         " given disk size, in MiB unless a suffix is used",
                         default=None, type="unit", metavar="<size>")

IGNORE_CONSIST_OPT = cli_option("--ignore-consistency",
                                dest="ignore_consistency",
                                action="store_true", default=False,
                                help="Ignore the consistency of the disks on"
                                " the secondary")

NONLIVE_OPT = cli_option("--non-live", dest="live",
                         default=True, action="store_false",
                         help="Do a non-live migration (this usually means"
                         " freeze the instance, save the state, transfer and"
                         " only then resume running on the secondary node)")

NODE_PLACEMENT_OPT = cli_option("-n", "--node", dest="node",
                                help="Target node and optional secondary node",
                                metavar="<pnode>[:<snode>]",
                                completion_suggest=OPT_COMPL_INST_ADD_NODES)

NODE_LIST_OPT = cli_option("-n", "--node", dest="nodes", default=[],
                           action="append", metavar="<node>",
                           help="Use only this node (can be used multiple"
                           " times, if not given defaults to all nodes)",
                           completion_suggest=OPT_COMPL_ONE_NODE)

SINGLE_NODE_OPT = cli_option("-n", "--node", dest="node", help="Target node",
                             metavar="<node>",
                             completion_suggest=OPT_COMPL_ONE_NODE)

NOSTART_OPT = cli_option("--no-start", dest="start", default=True,
                         action="store_false",
                         help="Don't start the instance after creation")

SHOWCMD_OPT = cli_option("--show-cmd", dest="show_command",
                         action="store_true", default=False,
                         help="Show command instead of executing it")

CLEANUP_OPT = cli_option("--cleanup", dest="cleanup",
                         default=False, action="store_true",
                         help="Instead of performing the migration, try to"
                         " recover from a failed cleanup. This is safe"
                         " to run even if the instance is healthy, but it"
                         " will create extra replication traffic and "
                         " disrupt briefly the replication (like during the"
                         " migration")

STATIC_OPT = cli_option("-s", "--static", dest="static",
                        action="store_true", default=False,
                        help="Only show configuration data, not runtime data")

ALL_OPT = cli_option("--all", dest="show_all",
                     default=False, action="store_true",
                     help="Show info on all instances on the cluster."
                     " This can take a long time to run, use wisely")

SELECT_OS_OPT = cli_option("--select-os", dest="select_os",
                           action="store_true", default=False,
                           help="Interactive OS reinstall, lists available"
                           " OS templates for selection")

IGNORE_FAILURES_OPT = cli_option("--ignore-failures", dest="ignore_failures",
                                 action="store_true", default=False,
                                 help="Remove the instance from the cluster"
                                 " configuration even if there are failures"
                                 " during the removal process")

NEW_SECONDARY_OPT = cli_option("-n", "--new-secondary", dest="dst_node",
                               help="Specifies the new secondary node",
                               metavar="NODE", default=None,
                               completion_suggest=OPT_COMPL_ONE_NODE)

ON_PRIMARY_OPT = cli_option("-p", "--on-primary", dest="on_primary",
                            default=False, action="store_true",
                            help="Replace the disk(s) on the primary"
                            " node (only for the drbd template)")

ON_SECONDARY_OPT = cli_option("-s", "--on-secondary", dest="on_secondary",
                              default=False, action="store_true",
                              help="Replace the disk(s) on the secondary"
                              " node (only for the drbd template)")

AUTO_PROMOTE_OPT = cli_option("--auto-promote", dest="auto_promote",
                              default=False, action="store_true",
                              help="Lock all nodes and auto-promote as needed"
                              " to MC status")

AUTO_REPLACE_OPT = cli_option("-a", "--auto", dest="auto",
                              default=False, action="store_true",
                              help="Automatically replace faulty disks"
                              " (only for the drbd template)")

IGNORE_SIZE_OPT = cli_option("--ignore-size", dest="ignore_size",
                             default=False, action="store_true",
                             help="Ignore current recorded size"
                             " (useful for forcing activation when"
                             " the recorded size is wrong)")

SRC_NODE_OPT = cli_option("--src-node", dest="src_node", help="Source node",
                          metavar="<node>",
                          completion_suggest=OPT_COMPL_ONE_NODE)

SRC_DIR_OPT = cli_option("--src-dir", dest="src_dir", help="Source directory",
                         metavar="<dir>")

SECONDARY_IP_OPT = cli_option("-s", "--secondary-ip", dest="secondary_ip",
                              help="Specify the secondary ip for the node",
                              metavar="ADDRESS", default=None)

READD_OPT = cli_option("--readd", dest="readd",
                       default=False, action="store_true",
                       help="Readd old node after replacing it")

NOSSH_KEYCHECK_OPT = cli_option("--no-ssh-key-check", dest="ssh_key_check",
                                default=True, action="store_false",
                                help="Disable SSH key fingerprint checking")


MC_OPT = cli_option("-C", "--master-candidate", dest="master_candidate",
                    choices=_YESNO, default=None, metavar=_YORNO,
                    help="Set the master_candidate flag on the node")

OFFLINE_OPT = cli_option("-O", "--offline", dest="offline", metavar=_YORNO,
                         choices=_YESNO, default=None,
                         help="Set the offline flag on the node")

DRAINED_OPT = cli_option("-D", "--drained", dest="drained", metavar=_YORNO,
                         choices=_YESNO, default=None,
                         help="Set the drained flag on the node")

ALLOCATABLE_OPT = cli_option("--allocatable", dest="allocatable",
                             choices=_YESNO, default=None, metavar=_YORNO,
                             help="Set the allocatable flag on a volume")

NOLVM_STORAGE_OPT = cli_option("--no-lvm-storage", dest="lvm_storage",
                               help="Disable support for lvm based instances"
                               " (cluster-wide)",
                               action="store_false", default=True)

ENABLED_HV_OPT = cli_option("--enabled-hypervisors",
                            dest="enabled_hypervisors",
                            help="Comma-separated list of hypervisors",
                            type="string", default=None)

NIC_PARAMS_OPT = cli_option("-N", "--nic-parameters", dest="nicparams",
                            type="keyval", default={},
                            help="NIC parameters")

CP_SIZE_OPT = cli_option("-C", "--candidate-pool-size", default=None,
                         dest="candidate_pool_size", type="int",
                         help="Set the candidate pool size")

VG_NAME_OPT = cli_option("-g", "--vg-name", dest="vg_name",
                         help="Enables LVM and specifies the volume group"
                         " name (cluster-wide) for disk allocation [xenvg]",
                         metavar="VG", default=None)

YES_DOIT_OPT = cli_option("--yes-do-it", dest="yes_do_it",
                          help="Destroy cluster", action="store_true")

NOVOTING_OPT = cli_option("--no-voting", dest="no_voting",
                          help="Skip node agreement check (dangerous)",
                          action="store_true", default=False)

MAC_PREFIX_OPT = cli_option("-m", "--mac-prefix", dest="mac_prefix",
                            help="Specify the mac prefix for the instance IP"
                            " addresses, in the format XX:XX:XX",
                            metavar="PREFIX",
                            default=None)

MASTER_NETDEV_OPT = cli_option("--master-netdev", dest="master_netdev",
                               help="Specify the node interface (cluster-wide)"
                               " on which the master IP address will be added "
                               " [%s]" % constants.DEFAULT_BRIDGE,
                               metavar="NETDEV",
                               default=constants.DEFAULT_BRIDGE)


GLOBAL_FILEDIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir",
                                help="Specify the default directory (cluster-"
                                "wide) for storing the file-based disks [%s]" %
                                constants.DEFAULT_FILE_STORAGE_DIR,
                                metavar="DIR",
                                default=constants.DEFAULT_FILE_STORAGE_DIR)

NOMODIFY_ETCHOSTS_OPT = cli_option("--no-etc-hosts", dest="modify_etc_hosts",
                                   help="Don't modify /etc/hosts",
                                   action="store_false", default=True)

NOMODIFY_SSH_SETUP_OPT = cli_option("--no-ssh-init", dest="modify_ssh_setup",
                                    help="Don't initialize SSH keys",
                                    action="store_false", default=True)

ERROR_CODES_OPT = cli_option("--error-codes", dest="error_codes",
                             help="Enable parseable error messages",
                             action="store_true", default=False)

NONPLUS1_OPT = cli_option("--no-nplus1-mem", dest="skip_nplusone_mem",
                          help="Skip N+1 memory redundancy tests",
                          action="store_true", default=False)

REBOOT_TYPE_OPT = cli_option("-t", "--type", dest="reboot_type",
                             help="Type of reboot: soft/hard/full",
                             default=constants.INSTANCE_REBOOT_HARD,
                             metavar="<REBOOT>",
                             choices=list(constants.REBOOT_TYPES))

IGNORE_SECONDARIES_OPT = cli_option("--ignore-secondaries",
                                    dest="ignore_secondaries",
                                    default=False, action="store_true",
                                    help="Ignore errors from secondaries")

NOSHUTDOWN_OPT = cli_option("--noshutdown", dest="shutdown",
                            action="store_false", default=True,
                            help="Don't shutdown the instance (unsafe)")

TIMEOUT_OPT = cli_option("--timeout", dest="timeout", type="int",
                         default=constants.DEFAULT_SHUTDOWN_TIMEOUT,
                         help="Maximum time to wait")

SHUTDOWN_TIMEOUT_OPT = cli_option("--shutdown-timeout",
                         dest="shutdown_timeout", type="int",
                         default=constants.DEFAULT_SHUTDOWN_TIMEOUT,
                         help="Maximum time to wait for instance shutdown")

EARLY_RELEASE_OPT = cli_option("--early-release",
                               dest="early_release", default=False,
                               action="store_true",
                               help="Release the locks on the secondary"
                               " node(s) early")

NEW_CLUSTER_CERT_OPT = cli_option("--new-cluster-certificate",
                                  dest="new_cluster_cert",
                                  default=False, action="store_true",
                                  help="Generate a new cluster certificate")

RAPI_CERT_OPT = cli_option("--rapi-certificate", dest="rapi_cert",
                           default=None,
                           help="File containing new RAPI certificate")

NEW_RAPI_CERT_OPT = cli_option("--new-rapi-certificate", dest="new_rapi_cert",
                               default=None, action="store_true",
                               help=("Generate a new self-signed RAPI"
                                     " certificate"))

NEW_CONFD_HMAC_KEY_OPT = cli_option("--new-confd-hmac-key",
                                    dest="new_confd_hmac_key",
                                    default=False, action="store_true",
                                    help=("Create a new HMAC key for %s" %
                                          constants.CONFD))


def _ParseArgs(argv, commands, aliases):
  """Parser for the command line arguments.

  This function parses the arguments and returns the function which
  must be executed together with its (modified) arguments.

  @param argv: the command line
  @param commands: dictionary with special contents, see the design
      doc for cmdline handling
  @param aliases: dictionary with command aliases {'alias': 'target, ...}

  """
  if len(argv) == 0:
    binary = "<command>"
  else:
    binary = argv[0].split("/")[-1]

  if len(argv) > 1 and argv[1] == "--version":
    ToStdout("%s (ganeti) %s", binary, constants.RELEASE_VERSION)
    # Quit right away. That way we don't have to care about this special
    # argument. optparse.py does it the same.
    sys.exit(0)

  if len(argv) < 2 or not (argv[1] in commands or
                           argv[1] in aliases):
    # let's do a nice thing
    sortedcmds = commands.keys()
    sortedcmds.sort()

    ToStdout("Usage: %s {command} [options...] [argument...]", binary)
    ToStdout("%s <command> --help to see details, or man %s", binary, binary)
    ToStdout("")

    # compute the max line length for cmd + usage
    mlen = max([len(" %s" % cmd) for cmd in commands])
    mlen = min(60, mlen) # should not get here...

    # and format a nice command list
    ToStdout("Commands:")
    for cmd in sortedcmds:
      cmdstr = " %s" % (cmd,)
      help_text = commands[cmd][4]
      help_lines = textwrap.wrap(help_text, 79 - 3 - mlen)
      ToStdout("%-*s - %s", mlen, cmdstr, help_lines.pop(0))
      for line in help_lines:
        ToStdout("%-*s   %s", mlen, "", line)

    ToStdout("")

    return None, None, None

  # get command, unalias it, and look it up in commands
  cmd = argv.pop(1)
  if cmd in aliases:
    if cmd in commands:
      raise errors.ProgrammerError("Alias '%s' overrides an existing"
                                   " command" % cmd)

    if aliases[cmd] not in commands:
      raise errors.ProgrammerError("Alias '%s' maps to non-existing"
                                   " command '%s'" % (cmd, aliases[cmd]))

    cmd = aliases[cmd]

  func, args_def, parser_opts, usage, description = commands[cmd]
  parser = OptionParser(option_list=parser_opts + [_DRY_RUN_OPT, DEBUG_OPT],
                        description=description,
                        formatter=TitledHelpFormatter(),
                        usage="%%prog %s %s" % (cmd, usage))
  parser.disable_interspersed_args()
  options, args = parser.parse_args()

  if not _CheckArguments(cmd, args_def, args):
    return None, None, None

  return func, options, args


def _CheckArguments(cmd, args_def, args):
  """Verifies the arguments using the argument definition.

  Algorithm:

    1. Abort with error if values specified by user but none expected.

    1. For each argument in definition

      1. Keep running count of minimum number of values (min_count)
      1. Keep running count of maximum number of values (max_count)
      1. If it has an unlimited number of values

        1. Abort with error if it's not the last argument in the definition

    1. If last argument has limited number of values

      1. Abort with error if number of values doesn't match or is too large

    1. Abort with error if user didn't pass enough values (min_count)

  """
  if args and not args_def:
    ToStderr("Error: Command %s expects no arguments", cmd)
    return False

  min_count = None
  max_count = None
  check_max = None

  last_idx = len(args_def) - 1

  for idx, arg in enumerate(args_def):
    if min_count is None:
      min_count = arg.min
    elif arg.min is not None:
      min_count += arg.min

    if max_count is None:
      max_count = arg.max
    elif arg.max is not None:
      max_count += arg.max

    if idx == last_idx:
      check_max = (arg.max is not None)

    elif arg.max is None:
      raise errors.ProgrammerError("Only the last argument can have max=None")

  if check_max:
    # Command with exact number of arguments
    if (min_count is not None and max_count is not None and
        min_count == max_count and len(args) != min_count):
      ToStderr("Error: Command %s expects %d argument(s)", cmd, min_count)
      return False

    # Command with limited number of arguments
    if max_count is not None and len(args) > max_count:
      ToStderr("Error: Command %s expects only %d argument(s)",
               cmd, max_count)
      return False

  # Command with some required arguments
  if min_count is not None and len(args) < min_count:
    ToStderr("Error: Command %s expects at least %d argument(s)",
             cmd, min_count)
    return False

  return True


def SplitNodeOption(value):
  """Splits the value of a --node option.

  """
  if value and ':' in value:
    return value.split(':', 1)
  else:
    return (value, None)


def CalculateOSNames(os_name, os_variants):
  """Calculates all the names an OS can be called, according to its variants.

  @type os_name: string
  @param os_name: base name of the os
  @type os_variants: list or None
  @param os_variants: list of supported variants
  @rtype: list
  @return: list of valid names

  """
  if os_variants:
    return ['%s+%s' % (os_name, v) for v in os_variants]
  else:
    return [os_name]


def UsesRPC(fn):
  def wrapper(*args, **kwargs):
    rpc.Init()
    try:
      return fn(*args, **kwargs)
    finally:
      rpc.Shutdown()
  return wrapper


def AskUser(text, choices=None):
  """Ask the user a question.

  @param text: the question to ask

  @param choices: list with elements tuples (input_char, return_value,
      description); if not given, it will default to: [('y', True,
      'Perform the operation'), ('n', False, 'Do no do the operation')];
      note that the '?' char is reserved for help

  @return: one of the return values from the choices list; if input is
      not possible (i.e. not running with a tty, we return the last
      entry from the list

  """
  if choices is None:
    choices = [('y', True, 'Perform the operation'),
               ('n', False, 'Do not perform the operation')]
  if not choices or not isinstance(choices, list):
    raise errors.ProgrammerError("Invalid choices argument to AskUser")
  for entry in choices:
    if not isinstance(entry, tuple) or len(entry) < 3 or entry[0] == '?':
      raise errors.ProgrammerError("Invalid choices element to AskUser")

  answer = choices[-1][1]
  new_text = []
  for line in text.splitlines():
    new_text.append(textwrap.fill(line, 70, replace_whitespace=False))
  text = "\n".join(new_text)
  try:
    f = file("/dev/tty", "a+")
  except IOError:
    return answer
  try:
    chars = [entry[0] for entry in choices]
    chars[-1] = "[%s]" % chars[-1]
    chars.append('?')
    maps = dict([(entry[0], entry[1]) for entry in choices])
    while True:
      f.write(text)
      f.write('\n')
      f.write("/".join(chars))
      f.write(": ")
      line = f.readline(2).strip().lower()
      if line in maps:
        answer = maps[line]
        break
      elif line == '?':
        for entry in choices:
          f.write(" %s - %s\n" % (entry[0], entry[2]))
        f.write("\n")
        continue
  finally:
    f.close()
  return answer


class JobSubmittedException(Exception):
  """Job was submitted, client should exit.

  This exception has one argument, the ID of the job that was
  submitted. The handler should print this ID.

  This is not an error, just a structured way to exit from clients.

  """


def SendJob(ops, cl=None):
  """Function to submit an opcode without waiting for the results.

  @type ops: list
  @param ops: list of opcodes
  @type cl: luxi.Client
  @param cl: the luxi client to use for communicating with the master;
             if None, a new client will be created

  """
  if cl is None:
    cl = GetClient()

  job_id = cl.SubmitJob(ops)

  return job_id


def PollJob(job_id, cl=None, feedback_fn=None):
  """Function to poll for the result of a job.

  @type job_id: job identified
  @param job_id: the job to poll for results
  @type cl: luxi.Client
  @param cl: the luxi client to use for communicating with the master;
             if None, a new client will be created

  """
  if cl is None:
    cl = GetClient()

  prev_job_info = None
  prev_logmsg_serial = None

  status = None

  notified_queued = False
  notified_waitlock = False

  while True:
    result = cl.WaitForJobChangeOnce(job_id, ["status"], prev_job_info,
                                     prev_logmsg_serial)
    if not result:
      # job not found, go away!
      raise errors.JobLost("Job with id %s lost" % job_id)
    elif result == constants.JOB_NOTCHANGED:
      if status is not None and not callable(feedback_fn):
        if status == constants.JOB_STATUS_QUEUED and not notified_queued:
          ToStderr("Job %s is waiting in queue", job_id)
          notified_queued = True
        elif status == constants.JOB_STATUS_WAITLOCK and not notified_waitlock:
          ToStderr("Job %s is trying to acquire all necessary locks", job_id)
          notified_waitlock = True

      # Wait again
      continue

    # Split result, a tuple of (field values, log entries)
    (job_info, log_entries) = result
    (status, ) = job_info

    if log_entries:
      for log_entry in log_entries:
        (serial, timestamp, _, message) = log_entry
        if callable(feedback_fn):
          feedback_fn(log_entry[1:])
        else:
          encoded = utils.SafeEncode(message)
          ToStdout("%s %s", time.ctime(utils.MergeTime(timestamp)), encoded)
        prev_logmsg_serial = max(prev_logmsg_serial, serial)

    # TODO: Handle canceled and archived jobs
    elif status in (constants.JOB_STATUS_SUCCESS,
                    constants.JOB_STATUS_ERROR,
                    constants.JOB_STATUS_CANCELING,
                    constants.JOB_STATUS_CANCELED):
      break

    prev_job_info = job_info

  jobs = cl.QueryJobs([job_id], ["status", "opstatus", "opresult"])
  if not jobs:
    raise errors.JobLost("Job with id %s lost" % job_id)

  status, opstatus, result = jobs[0]
  if status == constants.JOB_STATUS_SUCCESS:
    return result
  elif status in (constants.JOB_STATUS_CANCELING,
                  constants.JOB_STATUS_CANCELED):
    raise errors.OpExecError("Job was canceled")
  else:
    has_ok = False
    for idx, (status, msg) in enumerate(zip(opstatus, result)):
      if status == constants.OP_STATUS_SUCCESS:
        has_ok = True
      elif status == constants.OP_STATUS_ERROR:
        errors.MaybeRaise(msg)
        if has_ok:
          raise errors.OpExecError("partial failure (opcode %d): %s" %
                                   (idx, msg))
        else:
          raise errors.OpExecError(str(msg))
    # default failure mode
    raise errors.OpExecError(result)


def SubmitOpCode(op, cl=None, feedback_fn=None, opts=None):
  """Legacy function to submit an opcode.

  This is just a simple wrapper over the construction of the processor
  instance. It should be extended to better handle feedback and
  interaction functions.

  """
  if cl is None:
    cl = GetClient()

  SetGenericOpcodeOpts([op], opts)

  job_id = SendJob([op], cl)

  op_results = PollJob(job_id, cl=cl, feedback_fn=feedback_fn)

  return op_results[0]


def SubmitOrSend(op, opts, cl=None, feedback_fn=None):
  """Wrapper around SubmitOpCode or SendJob.

  This function will decide, based on the 'opts' parameter, whether to
  submit and wait for the result of the opcode (and return it), or
  whether to just send the job and print its identifier. It is used in
  order to simplify the implementation of the '--submit' option.

  It will also process the opcodes if we're sending the via SendJob
  (otherwise SubmitOpCode does it).

  """
  if opts and opts.submit_only:
    job = [op]
    SetGenericOpcodeOpts(job, opts)
    job_id = SendJob(job, cl=cl)
    raise JobSubmittedException(job_id)
  else:
    return SubmitOpCode(op, cl=cl, feedback_fn=feedback_fn, opts=opts)


def SetGenericOpcodeOpts(opcode_list, options):
  """Processor for generic options.

  This function updates the given opcodes based on generic command
  line options (like debug, dry-run, etc.).

  @param opcode_list: list of opcodes
  @param options: command line options or None
  @return: None (in-place modification)

  """
  if not options:
    return
  for op in opcode_list:
    op.dry_run = options.dry_run
    op.debug_level = options.debug