backend.py 82.2 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
#
Iustin Pop's avatar
Iustin Pop committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


22
23
24
25
26
27
"""Functions used by the node daemon

@var _ALLOWED_UPLOAD_FILES: denotes which files are accepted in
     the L{UploadFile} function

"""
Iustin Pop's avatar
Iustin Pop committed
28
29
30
31
32
33
34
35
36
37


import os
import os.path
import shutil
import time
import stat
import errno
import re
import subprocess
38
import random
39
import logging
40
import tempfile
41
42
import zlib
import base64
Iustin Pop's avatar
Iustin Pop committed
43
44
45
46
47
48
49
50

from ganeti import errors
from ganeti import utils
from ganeti import ssh
from ganeti import hypervisor
from ganeti import constants
from ganeti import bdev
from ganeti import objects
51
from ganeti import ssconf
Iustin Pop's avatar
Iustin Pop committed
52
53


Michael Hanselmann's avatar
Michael Hanselmann committed
54
def _GetConfig():
Iustin Pop's avatar
Iustin Pop committed
55
  """Simple wrapper to return a SimpleStore.
Iustin Pop's avatar
Iustin Pop committed
56

Iustin Pop's avatar
Iustin Pop committed
57
58
  @rtype: L{ssconf.SimpleStore}
  @return: a SimpleStore instance
Iustin Pop's avatar
Iustin Pop committed
59
60

  """
Iustin Pop's avatar
Iustin Pop committed
61
  return ssconf.SimpleStore()
Michael Hanselmann's avatar
Michael Hanselmann committed
62
63


64
def _GetSshRunner(cluster_name):
Iustin Pop's avatar
Iustin Pop committed
65
66
67
68
69
70
71
72
73
  """Simple wrapper to return an SshRunner.

  @type cluster_name: str
  @param cluster_name: the cluster name, which is needed
      by the SshRunner constructor
  @rtype: L{ssh.SshRunner}
  @return: an SshRunner instance

  """
74
  return ssh.SshRunner(cluster_name)
75
76


77
78
79
80
81
82
83
84
85
def _Decompress(data):
  """Unpacks data compressed by the RPC client.

  @type data: list or tuple
  @param data: Data sent by RPC client
  @rtype: str
  @return: Decompressed data

  """
86
  assert isinstance(data, (list, tuple))
87
88
89
90
91
92
93
94
95
96
  assert len(data) == 2
  (encoding, content) = data
  if encoding == constants.RPC_ENCODING_NONE:
    return content
  elif encoding == constants.RPC_ENCODING_ZLIB_BASE64:
    return zlib.decompress(base64.b64decode(content))
  else:
    raise AssertionError("Unknown data encoding")


97
def _CleanDirectory(path, exclude=None):
98
99
  """Removes all regular files in a directory.

Iustin Pop's avatar
Iustin Pop committed
100
101
  @type path: str
  @param path: the directory to clean
102
  @type exclude: list
Iustin Pop's avatar
Iustin Pop committed
103
104
  @param exclude: list of files to be excluded, defaults
      to the empty list
105
106

  """
107
108
  if not os.path.isdir(path):
    return
109
110
111
112
113
  if exclude is None:
    exclude = []
  else:
    # Normalize excluded paths
    exclude = [os.path.normpath(i) for i in exclude]
114

115
  for rel_name in utils.ListVisibleFiles(path):
116
117
118
    full_name = os.path.normpath(os.path.join(path, rel_name))
    if full_name in exclude:
      continue
119
120
121
122
    if os.path.isfile(full_name) and not os.path.islink(full_name):
      utils.RemoveFile(full_name)


123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def _BuildUploadFileList():
  """Build the list of allowed upload files.

  This is abstracted so that it's built only once at module import time.

  """
  return frozenset([
      constants.CLUSTER_CONF_FILE,
      constants.ETC_HOSTS,
      constants.SSH_KNOWN_HOSTS_FILE,
      constants.VNC_PASSWORD_FILE,
      ])


_ALLOWED_UPLOAD_FILES = _BuildUploadFileList()


140
def JobQueuePurge():
Iustin Pop's avatar
Iustin Pop committed
141
142
143
  """Removes job queue files and archived jobs.

  @rtype: None
144
145

  """
146
  _CleanDirectory(constants.QUEUE_DIR, exclude=[constants.JOB_QUEUE_LOCK_FILE])
147
148
149
  _CleanDirectory(constants.JOB_QUEUE_ARCHIVE_DIR)


150
151
152
153
154
155
156
def GetMasterInfo():
  """Returns master information.

  This is an utility function to compute master information, either
  for consumption here or from the node daemon.

  @rtype: tuple
Iustin Pop's avatar
Iustin Pop committed
157
158
  @return: (master_netdev, master_ip, master_name) if we have a good
      configuration, otherwise (None, None, None)
159
160
161

  """
  try:
Michael Hanselmann's avatar
Michael Hanselmann committed
162
163
164
165
    cfg = _GetConfig()
    master_netdev = cfg.GetMasterNetdev()
    master_ip = cfg.GetMasterIP()
    master_node = cfg.GetMasterNode()
Michael Hanselmann's avatar
Michael Hanselmann committed
166
  except errors.ConfigurationError:
167
    logging.exception("Cluster configuration incomplete")
Iustin Pop's avatar
Iustin Pop committed
168
    return (None, None, None)
169
  return (master_netdev, master_ip, master_node)
170
171


172
def StartMaster(start_daemons, no_voting):
Iustin Pop's avatar
Iustin Pop committed
173
174
  """Activate local node as master node.

175
  The function will always try activate the IP address of the master
Iustin Pop's avatar
Iustin Pop committed
176
177
178
179
180
181
  (unless someone else has it). It will also start the master daemons,
  based on the start_daemons parameter.

  @type start_daemons: boolean
  @param start_daemons: whther to also start the master
      daemons (ganeti-masterd and ganeti-rapi)
182
183
184
  @type no_voting: boolean
  @param no_voting: whether to start ganeti-masterd without a node vote
      (if start_daemons is True), but still non-interactively
Iustin Pop's avatar
Iustin Pop committed
185
  @rtype: None
Iustin Pop's avatar
Iustin Pop committed
186
187

  """
188
  ok = True
189
  master_netdev, master_ip, _ = GetMasterInfo()
190
  if not master_netdev:
Iustin Pop's avatar
Iustin Pop committed
191
192
    return False

193
  if utils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
194
    if utils.OwnIpAddress(master_ip):
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
      # we already have the ip:
      logging.debug("Already started")
    else:
      logging.error("Someone else has the master ip, not activating")
      ok = False
  else:
    result = utils.RunCmd(["ip", "address", "add", "%s/32" % master_ip,
                           "dev", master_netdev, "label",
                           "%s:0" % master_netdev])
    if result.failed:
      logging.error("Can't activate master IP: %s", result.output)
      ok = False

    result = utils.RunCmd(["arping", "-q", "-U", "-c 3", "-I", master_netdev,
                           "-s", master_ip, master_ip])
    # we'll ignore the exit code of arping

  # and now start the master and rapi daemons
  if start_daemons:
214
215
216
217
218
219
220
221
222
223
224
    daemons_params = {
        'ganeti-masterd': [],
        'ganeti-rapi': [],
        }
    if no_voting:
      daemons_params['ganeti-masterd'].append('--no-voting')
      daemons_params['ganeti-masterd'].append('--yes-do-it')
    for daemon in daemons_params:
      cmd = [daemon]
      cmd.extend(daemons_params[daemon])
      result = utils.RunCmd(cmd)
225
226
227
228
      if result.failed:
        logging.error("Can't start daemon %s: %s", daemon, result.output)
        ok = False
  return ok
Iustin Pop's avatar
Iustin Pop committed
229
230


231
def StopMaster(stop_daemons):
Iustin Pop's avatar
Iustin Pop committed
232
233
  """Deactivate this node as master.

234
  The function will always try to deactivate the IP address of the
Iustin Pop's avatar
Iustin Pop committed
235
236
237
238
239
240
241
  master. It will also stop the master daemons depending on the
  stop_daemons parameter.

  @type stop_daemons: boolean
  @param stop_daemons: whether to also stop the master daemons
      (ganeti-masterd and ganeti-rapi)
  @rtype: None
Iustin Pop's avatar
Iustin Pop committed
242
243

  """
244
  master_netdev, master_ip, _ = GetMasterInfo()
245
246
  if not master_netdev:
    return False
Iustin Pop's avatar
Iustin Pop committed
247

248
249
  result = utils.RunCmd(["ip", "address", "del", "%s/32" % master_ip,
                         "dev", master_netdev])
Iustin Pop's avatar
Iustin Pop committed
250
  if result.failed:
251
    logging.error("Can't remove the master IP, error: %s", result.output)
252
253
254
255
256
257
    # but otherwise ignore the failure

  if stop_daemons:
    # stop/kill the rapi and the master daemon
    for daemon in constants.RAPI_PID, constants.MASTERD_PID:
      utils.KillProcess(utils.ReadPidFile(utils.DaemonPidFileName(daemon)))
Iustin Pop's avatar
Iustin Pop committed
258
259
260
261

  return True


Iustin Pop's avatar
Iustin Pop committed
262
def AddNode(dsa, dsapub, rsa, rsapub, sshkey, sshpub):
263
  """Joins this node to the cluster.
Iustin Pop's avatar
Iustin Pop committed
264

265
266
267
268
  This does the following:
      - updates the hostkeys of the machine (rsa and dsa)
      - adds the ssh private key to the user
      - adds the ssh public key to the users' authorized_keys file
Iustin Pop's avatar
Iustin Pop committed
269

Iustin Pop's avatar
Iustin Pop committed
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
  @type dsa: str
  @param dsa: the DSA private key to write
  @type dsapub: str
  @param dsapub: the DSA public key to write
  @type rsa: str
  @param rsa: the RSA private key to write
  @type rsapub: str
  @param rsapub: the RSA public key to write
  @type sshkey: str
  @param sshkey: the SSH private key to write
  @type sshpub: str
  @param sshpub: the SSH public key to write
  @rtype: boolean
  @return: the success of the operation

285
  """
286
287
288
289
  sshd_keys =  [(constants.SSH_HOST_RSA_PRIV, rsa, 0600),
                (constants.SSH_HOST_RSA_PUB, rsapub, 0644),
                (constants.SSH_HOST_DSA_PRIV, dsa, 0600),
                (constants.SSH_HOST_DSA_PUB, dsapub, 0644)]
290
  for name, content, mode in sshd_keys:
291
    utils.WriteFile(name, data=content, mode=mode)
Iustin Pop's avatar
Iustin Pop committed
292

293
294
295
296
  try:
    priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS,
                                                    mkdir=True)
  except errors.OpExecError, err:
297
298
299
    msg = "Error while processing user ssh files"
    logging.exception(msg)
    return (False, "%s: %s" % (msg, err))
Iustin Pop's avatar
Iustin Pop committed
300

301
302
  for name, content in [(priv_key, sshkey), (pub_key, sshpub)]:
    utils.WriteFile(name, data=content, mode=0600)
Iustin Pop's avatar
Iustin Pop committed
303

304
  utils.AddAuthorizedKey(auth_keys, sshpub)
Iustin Pop's avatar
Iustin Pop committed
305

306
  utils.RunCmd([constants.SSH_INITD_SCRIPT, "restart"])
Iustin Pop's avatar
Iustin Pop committed
307

308
  return (True, "Node added successfully")
Iustin Pop's avatar
Iustin Pop committed
309
310
311


def LeaveCluster():
Iustin Pop's avatar
Iustin Pop committed
312
313
314
315
316
317
  """Cleans up and remove the current node.

  This function cleans up and prepares the current node to be removed
  from the cluster.

  If processing is successful, then it raises an
Iustin Pop's avatar
Iustin Pop committed
318
  L{errors.QuitGanetiException} which is used as a special case to
Iustin Pop's avatar
Iustin Pop committed
319
  shutdown the node daemon.
Iustin Pop's avatar
Iustin Pop committed
320
321

  """
322
  _CleanDirectory(constants.DATA_DIR)
323
  JobQueuePurge()
324

325
326
  try:
    priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS)
327
328
  except errors.OpExecError:
    logging.exception("Error while processing ssh files")
329
330
    return

331
  f = open(pub_key, 'r')
Iustin Pop's avatar
Iustin Pop committed
332
  try:
333
    utils.RemoveAuthorizedKey(auth_keys, f.read(8192))
Iustin Pop's avatar
Iustin Pop committed
334
335
336
  finally:
    f.close()

337
338
  utils.RemoveFile(priv_key)
  utils.RemoveFile(pub_key)
Iustin Pop's avatar
Iustin Pop committed
339

340
341
342
  # Return a reassuring string to the caller, and quit
  raise errors.QuitGanetiException(False, 'Shutdown scheduled')

Iustin Pop's avatar
Iustin Pop committed
343

344
def GetNodeInfo(vgname, hypervisor_type):
Michael Hanselmann's avatar
Michael Hanselmann committed
345
  """Gives back a hash with different information about the node.
Iustin Pop's avatar
Iustin Pop committed
346

347
348
349
350
351
352
353
354
355
356
357
358
  @type vgname: C{string}
  @param vgname: the name of the volume group to ask for disk space information
  @type hypervisor_type: C{str}
  @param hypervisor_type: the name of the hypervisor to ask for
      memory information
  @rtype: C{dict}
  @return: dictionary with the following keys:
      - vg_size is the size of the configured volume group in MiB
      - vg_free is the free size of the volume group in MiB
      - memory_dom0 is the memory allocated for domain0 in MiB
      - memory_free is the currently available (free) ram in MiB
      - memory_total is the total number of ram in MiB
Iustin Pop's avatar
Iustin Pop committed
359

360
  """
Iustin Pop's avatar
Iustin Pop committed
361
362
363
364
365
  outputarray = {}
  vginfo = _GetVGInfo(vgname)
  outputarray['vg_size'] = vginfo['vg_size']
  outputarray['vg_free'] = vginfo['vg_free']

366
  hyper = hypervisor.GetHypervisor(hypervisor_type)
Iustin Pop's avatar
Iustin Pop committed
367
368
369
370
  hyp_info = hyper.GetNodeInfo()
  if hyp_info is not None:
    outputarray.update(hyp_info)

371
372
373
374
375
376
  f = open("/proc/sys/kernel/random/boot_id", 'r')
  try:
    outputarray["bootid"] = f.read(128).rstrip("\n")
  finally:
    f.close()

Iustin Pop's avatar
Iustin Pop committed
377
378
379
  return outputarray


380
def VerifyNode(what, cluster_name):
Iustin Pop's avatar
Iustin Pop committed
381
382
  """Verify the status of the local node.

383
384
385
386
387
388
389
390
391
  Based on the input L{what} parameter, various checks are done on the
  local node.

  If the I{filelist} key is present, this list of
  files is checksummed and the file/checksum pairs are returned.

  If the I{nodelist} key is present, we check that we have
  connectivity via ssh with the target nodes (and check the hostname
  report).
Iustin Pop's avatar
Iustin Pop committed
392

393
394
395
396
397
398
399
400
401
402
403
  If the I{node-net-test} key is present, we check that we have
  connectivity to the given nodes via both primary IP and, if
  applicable, secondary IPs.

  @type what: C{dict}
  @param what: a dictionary of things to check:
      - filelist: list of files for which to compute checksums
      - nodelist: list of nodes we should check ssh communication with
      - node-net-test: list of nodes we should check node daemon port
        connectivity with
      - hypervisor: list with hypervisors to run the verify for
Iustin Pop's avatar
Iustin Pop committed
404
405
406
  @rtype: dict
  @return: a dictionary with the same keys as the input dict, and
      values representing the result of the checks
Iustin Pop's avatar
Iustin Pop committed
407
408
409
410

  """
  result = {}

411
412
413
414
415
416
417
418
419
420
421
422
423
  if constants.NV_HYPERVISOR in what:
    result[constants.NV_HYPERVISOR] = tmp = {}
    for hv_name in what[constants.NV_HYPERVISOR]:
      tmp[hv_name] = hypervisor.GetHypervisor(hv_name).Verify()

  if constants.NV_FILELIST in what:
    result[constants.NV_FILELIST] = utils.FingerprintFiles(
      what[constants.NV_FILELIST])

  if constants.NV_NODELIST in what:
    result[constants.NV_NODELIST] = tmp = {}
    random.shuffle(what[constants.NV_NODELIST])
    for node in what[constants.NV_NODELIST]:
424
      success, message = _GetSshRunner(cluster_name).VerifyNodeHostname(node)
Iustin Pop's avatar
Iustin Pop committed
425
      if not success:
426
427
428
429
        tmp[node] = message

  if constants.NV_NODENETTEST in what:
    result[constants.NV_NODENETTEST] = tmp = {}
430
431
    my_name = utils.HostInfo().name
    my_pip = my_sip = None
432
    for name, pip, sip in what[constants.NV_NODENETTEST]:
433
434
435
436
437
      if name == my_name:
        my_pip = pip
        my_sip = sip
        break
    if not my_pip:
438
439
      tmp[my_name] = ("Can't find my own primary/secondary IP"
                      " in the node list")
440
    else:
Michael Hanselmann's avatar
Michael Hanselmann committed
441
      port = utils.GetNodeDaemonPort()
442
      for name, pip, sip in what[constants.NV_NODENETTEST]:
443
444
445
446
447
448
449
        fail = []
        if not utils.TcpPing(pip, port, source=my_pip):
          fail.append("primary")
        if sip != pip:
          if not utils.TcpPing(sip, port, source=my_sip):
            fail.append("secondary")
        if fail:
450
451
452
453
454
455
456
457
458
459
460
461
462
463
          tmp[name] = ("failure using the %s interface(s)" %
                       " and ".join(fail))

  if constants.NV_LVLIST in what:
    result[constants.NV_LVLIST] = GetVolumeList(what[constants.NV_LVLIST])

  if constants.NV_INSTANCELIST in what:
    result[constants.NV_INSTANCELIST] = GetInstanceList(
      what[constants.NV_INSTANCELIST])

  if constants.NV_VGLIST in what:
    result[constants.NV_VGLIST] = ListVolumeGroups()

  if constants.NV_VERSION in what:
464
465
    result[constants.NV_VERSION] = (constants.PROTOCOL_VERSION,
                                    constants.RELEASE_VERSION)
466
467
468
469

  if constants.NV_HVINFO in what:
    hyper = hypervisor.GetHypervisor(what[constants.NV_HVINFO])
    result[constants.NV_HVINFO] = hyper.GetNodeInfo()
470

471
472
473
  if constants.NV_DRBDLIST in what:
    try:
      used_minors = bdev.DRBD8.GetUsedDevs().keys()
474
    except errors.BlockDeviceError, err:
475
      logging.warning("Can't get used minors list", exc_info=True)
476
      used_minors = str(err)
477
478
    result[constants.NV_DRBDLIST] = used_minors

Iustin Pop's avatar
Iustin Pop committed
479
480
481
482
483
484
  return result


def GetVolumeList(vg_name):
  """Compute list of logical volumes and their size.

Iustin Pop's avatar
Iustin Pop committed
485
486
487
488
489
490
491
492
493
494
495
  @type vg_name: str
  @param vg_name: the volume group whose LVs we should list
  @rtype: dict
  @return:
      dictionary of all partions (key) with value being a tuple of
      their size (in MiB), inactive and online status::

        {'test1': ('20.06', True, True)}

      in case of errors, a string is returned with the error
      details.
Iustin Pop's avatar
Iustin Pop committed
496
497

  """
498
499
500
501
502
  lvs = {}
  sep = '|'
  result = utils.RunCmd(["lvs", "--noheadings", "--units=m", "--nosuffix",
                         "--separator=%s" % sep,
                         "-olv_name,lv_size,lv_attr", vg_name])
Iustin Pop's avatar
Iustin Pop committed
503
  if result.failed:
504
505
    logging.error("Failed to list logical volumes, lvs output: %s",
                  result.output)
506
    return result.output
507

508
  valid_line_re = re.compile("^ *([^|]+)\|([0-9.]+)\|([^|]{6})\|?$")
509
  for line in result.stdout.splitlines():
510
511
512
    line = line.strip()
    match = valid_line_re.match(line)
    if not match:
513
      logging.error("Invalid line returned from lvs output: '%s'", line)
514
515
      continue
    name, size, attr = match.groups()
516
517
518
519
520
    inactive = attr[4] == '-'
    online = attr[5] == 'o'
    lvs[name] = (size, inactive, online)

  return lvs
Iustin Pop's avatar
Iustin Pop committed
521
522
523


def ListVolumeGroups():
Alexander Schreiber's avatar
Alexander Schreiber committed
524
  """List the volume groups and their size.
Iustin Pop's avatar
Iustin Pop committed
525

Iustin Pop's avatar
Iustin Pop committed
526
527
528
  @rtype: dict
  @return: dictionary with keys volume name and values the
      size of the volume
Iustin Pop's avatar
Iustin Pop committed
529
530
531
532
533

  """
  return utils.ListVolumeGroups()


534
535
536
def NodeVolumes():
  """List all volumes on this node.

Iustin Pop's avatar
Iustin Pop committed
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
  @rtype: list
  @return:
    A list of dictionaries, each having four keys:
      - name: the logical volume name,
      - size: the size of the logical volume
      - dev: the physical device on which the LV lives
      - vg: the volume group to which it belongs

    In case of errors, we return an empty list and log the
    error.

    Note that since a logical volume can live on multiple physical
    volumes, the resulting list might include a logical volume
    multiple times.

552
553
554
555
556
  """
  result = utils.RunCmd(["lvs", "--noheadings", "--units=m", "--nosuffix",
                         "--separator=|",
                         "--options=lv_name,lv_size,devices,vg_name"])
  if result.failed:
557
558
    logging.error("Failed to list logical volumes, lvs output: %s",
                  result.output)
Iustin Pop's avatar
Iustin Pop committed
559
    return []
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574

  def parse_dev(dev):
    if '(' in dev:
      return dev.split('(')[0]
    else:
      return dev

  def map_line(line):
    return {
      'name': line[0].strip(),
      'size': line[1].strip(),
      'dev': parse_dev(line[2].strip()),
      'vg': line[3].strip(),
    }

575
576
  return [map_line(line.split('|')) for line in result.stdout.splitlines()
          if line.count('|') >= 3]
577
578


Iustin Pop's avatar
Iustin Pop committed
579
def BridgesExist(bridges_list):
Alexander Schreiber's avatar
Alexander Schreiber committed
580
  """Check if a list of bridges exist on the current node.
Iustin Pop's avatar
Iustin Pop committed
581

Iustin Pop's avatar
Iustin Pop committed
582
583
  @rtype: boolean
  @return: C{True} if all of them exist, C{False} otherwise
Iustin Pop's avatar
Iustin Pop committed
584
585
586
587
588
589
590
591
592

  """
  for bridge in bridges_list:
    if not utils.BridgeExists(bridge):
      return False

  return True


593
def GetInstanceList(hypervisor_list):
Alexander Schreiber's avatar
Alexander Schreiber committed
594
  """Provides a list of instances.
Iustin Pop's avatar
Iustin Pop committed
595

596
597
598
599
600
  @type hypervisor_list: list
  @param hypervisor_list: the list of hypervisors to query information

  @rtype: list
  @return: a list of all running instances on the current node
Iustin Pop's avatar
Iustin Pop committed
601
602
    - instance1.example.com
    - instance2.example.com
Iustin Pop's avatar
Iustin Pop committed
603

604
  """
605
606
607
608
609
  results = []
  for hname in hypervisor_list:
    try:
      names = hypervisor.GetHypervisor(hname).ListInstances()
      results.extend(names)
Michael Hanselmann's avatar
Michael Hanselmann committed
610
    except errors.HypervisorError:
611
612
      logging.exception("Error enumerating instances for hypevisor %s", hname)
      raise
Iustin Pop's avatar
Iustin Pop committed
613

614
  return results
Iustin Pop's avatar
Iustin Pop committed
615
616


617
def GetInstanceInfo(instance, hname):
Michael Hanselmann's avatar
Michael Hanselmann committed
618
  """Gives back the information about an instance as a dictionary.
Iustin Pop's avatar
Iustin Pop committed
619

620
621
622
623
  @type instance: string
  @param instance: the instance name
  @type hname: string
  @param hname: the hypervisor type of the instance
Iustin Pop's avatar
Iustin Pop committed
624

625
626
627
628
629
  @rtype: dict
  @return: dictionary with the following keys:
      - memory: memory size of instance (int)
      - state: xen state of instance (string)
      - time: cpu time of instance (float)
Iustin Pop's avatar
Iustin Pop committed
630

631
  """
Iustin Pop's avatar
Iustin Pop committed
632
633
  output = {}

634
  iinfo = hypervisor.GetHypervisor(hname).GetInstanceInfo(instance)
Iustin Pop's avatar
Iustin Pop committed
635
636
637
638
639
640
641
642
  if iinfo is not None:
    output['memory'] = iinfo[2]
    output['state'] = iinfo[4]
    output['time'] = iinfo[5]

  return output


643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
def GetInstanceMigratable(instance):
  """Gives whether an instance can be migrated.

  @type instance: L{objects.Instance}
  @param instance: object representing the instance to be checked.

  @rtype: tuple
  @return: tuple of (result, description) where:
      - result: whether the instance can be migrated or not
      - description: a description of the issue, if relevant

  """
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
  if instance.name not in hyper.ListInstances():
    return (False, 'not running')

  for idx in range(len(instance.disks)):
    link_name = _GetBlockDevSymlinkPath(instance.name, idx)
    if not os.path.islink(link_name):
      return (False, 'not restarted since ganeti 1.2.5')

  return (True, '')


667
def GetAllInstancesInfo(hypervisor_list):
Iustin Pop's avatar
Iustin Pop committed
668
669
  """Gather data about all instances.

Iustin Pop's avatar
Iustin Pop committed
670
  This is the equivalent of L{GetInstanceInfo}, except that it
Iustin Pop's avatar
Iustin Pop committed
671
672
673
  computes data for all instances at once, thus being faster if one
  needs data about more than one instance.

674
675
676
  @type hypervisor_list: list
  @param hypervisor_list: list of hypervisors to query for instance data

677
  @rtype: dict
678
679
680
681
  @return: dictionary of instance: data, with data having the following keys:
      - memory: memory size of instance (int)
      - state: xen state of instance (string)
      - time: cpu time of instance (float)
Iustin Pop's avatar
Iustin Pop committed
682
      - vcpus: the number of vcpus
Iustin Pop's avatar
Iustin Pop committed
683

684
  """
Iustin Pop's avatar
Iustin Pop committed
685
686
  output = {}

687
688
689
690
  for hname in hypervisor_list:
    iinfo = hypervisor.GetHypervisor(hname).GetAllInstancesInfo()
    if iinfo:
      for name, inst_id, memory, vcpus, state, times in iinfo:
691
        value = {
692
693
694
695
696
          'memory': memory,
          'vcpus': vcpus,
          'state': state,
          'time': times,
          }
697
698
699
700
701
702
703
704
        if name in output:
          # we only check static parameters, like memory and vcpus,
          # and not state and time which can change between the
          # invocations of the different hypervisors
          for key in 'memory', 'vcpus':
            if value[key] != output[name][key]:
              raise errors.HypervisorError("Instance %s is running twice"
                                           " with different parameters" % name)
705
        output[name] = value
Iustin Pop's avatar
Iustin Pop committed
706
707
708
709

  return output


710
def InstanceOsAdd(instance):
Alexander Schreiber's avatar
Alexander Schreiber committed
711
  """Add an OS to an instance.
Iustin Pop's avatar
Iustin Pop committed
712

713
714
  @type instance: L{objects.Instance}
  @param instance: Instance whose OS is to be installed
Iustin Pop's avatar
Iustin Pop committed
715
716
  @rtype: boolean
  @return: the success of the operation
Iustin Pop's avatar
Iustin Pop committed
717
718

  """
719
720
721
722
723
724
725
726
727
  try:
    inst_os = OSFromDisk(instance.os)
  except errors.InvalidOS, err:
    os_name, os_dir, os_err = err.args
    if os_dir is None:
      return (False, "Can't find OS '%s': %s" % (os_name, os_err))
    else:
      return (False, "Error parsing OS '%s' in directory %s: %s" %
              (os_name, os_dir, os_err))
Iustin Pop's avatar
Iustin Pop committed
728

729
  create_env = OSEnvironment(instance)
Iustin Pop's avatar
Iustin Pop committed
730
731
732

  logfile = "%s/add-%s-%s-%d.log" % (constants.LOG_OS_DIR, instance.os,
                                     instance.name, int(time.time()))
733

734
735
  result = utils.RunCmd([inst_os.create_script], env=create_env,
                        cwd=inst_os.path, output=logfile,)
736
  if result.failed:
737
    logging.error("os create command '%s' returned error: %s, logfile: %s,"
738
                  " output: %s", result.cmd, result.fail_reason, logfile,
739
                  result.output)
740
    lines = [utils.SafeEncode(val)
741
742
743
             for val in utils.TailFile(logfile, lines=20)]
    return (False, "OS create script failed (%s), last lines in the"
            " log file:\n%s" % (result.fail_reason, "\n".join(lines)))
744

745
  return (True, "Successfully installed")
746
747


748
def RunRenameInstance(instance, old_name):
749
750
  """Run the OS rename script for an instance.

Iustin Pop's avatar
Iustin Pop committed
751
  @type instance: L{objects.Instance}
752
753
754
  @param instance: Instance whose OS is to be installed
  @type old_name: string
  @param old_name: previous instance name
Iustin Pop's avatar
Iustin Pop committed
755
756
  @rtype: boolean
  @return: the success of the operation
757
758
759
760

  """
  inst_os = OSFromDisk(instance.os)

761
762
  rename_env = OSEnvironment(instance)
  rename_env['OLD_INSTANCE_NAME'] = old_name
763
764
765
766

  logfile = "%s/rename-%s-%s-%s-%d.log" % (constants.LOG_OS_DIR, instance.os,
                                           old_name,
                                           instance.name, int(time.time()))
Iustin Pop's avatar
Iustin Pop committed
767

768
769
  result = utils.RunCmd([inst_os.rename_script], env=rename_env,
                        cwd=inst_os.path, output=logfile)
Iustin Pop's avatar
Iustin Pop committed
770
771

  if result.failed:
772
    logging.error("os create command '%s' returned error: %s output: %s",
773
                  result.cmd, result.fail_reason, result.output)
774
    lines = [utils.SafeEncode(val)
775
776
777
             for val in utils.TailFile(logfile, lines=20)]
    return (False, "OS rename script failed (%s), last lines in the"
            " log file:\n%s" % (result.fail_reason, "\n".join(lines)))
Iustin Pop's avatar
Iustin Pop committed
778

779
  return (True, "Rename successful")
Iustin Pop's avatar
Iustin Pop committed
780
781
782


def _GetVGInfo(vg_name):
Michael Hanselmann's avatar
Michael Hanselmann committed
783
  """Get information about the volume group.
Iustin Pop's avatar
Iustin Pop committed
784

Iustin Pop's avatar
Iustin Pop committed
785
786
787
788
789
790
791
792
  @type vg_name: str
  @param vg_name: the volume group which we query
  @rtype: dict
  @return:
    A dictionary with the following keys:
      - C{vg_size} is the total size of the volume group in MiB
      - C{vg_free} is the free size of the volume group in MiB
      - C{pv_count} are the number of physical disks in that VG
Iustin Pop's avatar
Iustin Pop committed
793

Iustin Pop's avatar
Iustin Pop committed
794
795
    If an error occurs during gathering of data, we return the same dict
    with keys all set to None.
796

Iustin Pop's avatar
Iustin Pop committed
797
  """
798
799
  retdic = dict.fromkeys(["vg_size", "vg_free", "pv_count"])

Iustin Pop's avatar
Iustin Pop committed
800
801
802
803
  retval = utils.RunCmd(["vgs", "-ovg_size,vg_free,pv_count", "--noheadings",
                         "--nosuffix", "--units=m", "--separator=:", vg_name])

  if retval.failed:
804
    logging.error("volume group %s not present", vg_name)
805
    return retdic
Iustin Pop's avatar
Iustin Pop committed
806
  valarr = retval.stdout.strip().rstrip(':').split(':')
807
808
809
810
811
812
813
814
  if len(valarr) == 3:
    try:
      retdic = {
        "vg_size": int(round(float(valarr[0]), 0)),
        "vg_free": int(round(float(valarr[1]), 0)),
        "pv_count": int(valarr[2]),
        }
    except ValueError, err:
815
      logging.exception("Fail to parse vgs output")
816
  else:
817
818
    logging.error("vgs output has the wrong number of fields (expected"
                  " three): %s", str(valarr))
Iustin Pop's avatar
Iustin Pop committed
819
820
821
  return retdic


822
823
824
825
826
827
def _GetBlockDevSymlinkPath(instance_name, idx):
  return os.path.join(constants.DISK_LINKS_DIR,
                      "%s:%d" % (instance_name, idx))


def _SymlinkBlockDev(instance_name, device_path, idx):
828
829
830
831
832
833
  """Set up symlinks to a instance's block device.

  This is an auxiliary function run when an instance is start (on the primary
  node) or when an instance is migrated (on the target node).


834
835
836
837
  @param instance_name: the name of the target instance
  @param device_path: path of the physical block device, on the node
  @param idx: the disk index
  @return: absolute path to the disk's symlink
838
839

  """
840
  link_name = _GetBlockDevSymlinkPath(instance_name, idx)
841
842
  try:
    os.symlink(device_path, link_name)
843
844
  except OSError, err:
    if err.errno == errno.EEXIST:
845
846
847
848
849
850
851
852
853
854
      if (not os.path.islink(link_name) or
          os.readlink(link_name) != device_path):
        os.remove(link_name)
        os.symlink(device_path, link_name)
    else:
      raise

  return link_name


855
def _RemoveBlockDevLinks(instance_name, disks):
Iustin Pop's avatar
Iustin Pop committed
856
857
858
  """Remove the block device symlinks belonging to the given instance.

  """
859
860
861
  for idx, disk in enumerate(disks):
    link_name = _GetBlockDevSymlinkPath(instance_name, idx)
    if os.path.islink(link_name):
Iustin Pop's avatar
Iustin Pop committed
862
      try:
863
864
865
        os.remove(link_name)
      except OSError:
        logging.exception("Can't remove symlink '%s'", link_name)
Iustin Pop's avatar
Iustin Pop committed
866
867


868
def _GatherAndLinkBlockDevs(instance):
Iustin Pop's avatar
Iustin Pop committed
869
870
871
872
873
  """Set up an instance's block device(s).

  This is run on the primary node at instance startup. The block
  devices must be already assembled.

Iustin Pop's avatar
Iustin Pop committed
874
875
  @type instance: L{objects.Instance}
  @param instance: the instance whose disks we shoul assemble
876
877
  @rtype: list
  @return: list of (disk_object, device_path)
Iustin Pop's avatar
Iustin Pop committed
878

Iustin Pop's avatar
Iustin Pop committed
879
880
  """
  block_devices = []
881
  for idx, disk in enumerate(instance.disks):
Iustin Pop's avatar
Iustin Pop committed
882
883
884
885
886
    device = _RecursiveFindBD(disk)
    if device is None:
      raise errors.BlockDeviceError("Block device '%s' is not set up." %
                                    str(disk))
    device.Open()
887
    try:
888
      link_name = _SymlinkBlockDev(instance.name, device.dev_path, idx)
889
890
891
892
893
894
    except OSError, e:
      raise errors.BlockDeviceError("Cannot create block device symlink: %s" %
                                    e.strerror)

    block_devices.append((disk, link_name))

Iustin Pop's avatar
Iustin Pop committed
895
896
897
  return block_devices


898
def StartInstance(instance):
Iustin Pop's avatar
Iustin Pop committed
899
900
  """Start an instance.

Iustin Pop's avatar
Iustin Pop committed
901
  @type instance: L{objects.Instance}
902
903
904
  @param instance: the instance object
  @rtype: boolean
  @return: whether the startup was successful or not
Iustin Pop's avatar
Iustin Pop committed
905

906
  """
907
  running_instances = GetInstanceList([instance.hypervisor])
Iustin Pop's avatar
Iustin Pop committed
908
909

  if instance.name in running_instances:
910
    return (True, "Already running")
Iustin Pop's avatar
Iustin Pop committed
911
912

  try:
913
914
    block_devices = _GatherAndLinkBlockDevs(instance)
    hyper = hypervisor.GetHypervisor(instance.hypervisor)
915
    hyper.StartInstance(instance, block_devices)
916
917
  except errors.BlockDeviceError, err:
    logging.exception("Failed to start instance")
918
    return (False, "Block device error: %s" % str(err))
Iustin Pop's avatar
Iustin Pop committed
919
  except errors.HypervisorError, err:
920
    logging.exception("Failed to start instance")
921
    _RemoveBlockDevLinks(instance.name, instance.disks)
922
    return (False, "Hypervisor error: %s" % str(err))
Iustin Pop's avatar
Iustin Pop committed
923

924
  return (True, "Instance started successfully")
Iustin Pop's avatar
Iustin Pop committed
925
926


927
def InstanceShutdown(instance):
Iustin Pop's avatar
Iustin Pop committed
928
929
  """Shut an instance down.

Iustin Pop's avatar
Iustin Pop committed
930
931
932
  @note: this functions uses polling with a hardcoded timeout.

  @type instance: L{objects.Instance}
933
934
935
  @param instance: the instance object
  @rtype: boolean
  @return: whether the startup was successful or not
Iustin Pop's avatar
Iustin Pop committed
936

937
  """
938
939
  hv_name = instance.hypervisor
  running_instances = GetInstanceList([hv_name])
Iustin Pop's avatar
Iustin Pop committed
940
941

  if instance.name not in running_instances:
942
    return (True, "Instance already stopped")
Iustin Pop's avatar
Iustin Pop committed
943

944
  hyper = hypervisor.GetHypervisor(hv_name)
Iustin Pop's avatar
Iustin Pop committed
945
946
947
  try:
    hyper.StopInstance(instance)
  except errors.HypervisorError, err:
948
949
950
    msg = "Failed to stop instance %s: %s" % (instance.name, err)
    logging.error(msg)
    return (False, msg)
Iustin Pop's avatar
Iustin Pop committed
951
952
953
954

  # test every 10secs for 2min

  time.sleep(1)
Michael Hanselmann's avatar
Michael Hanselmann committed
955
  for _ in range(11):
956
    if instance.name not in GetInstanceList([hv_name]):
Iustin Pop's avatar
Iustin Pop committed
957
958
959
960
      break
    time.sleep(10)
  else:
    # the shutdown did not succeed
961
962
    logging.error("Shutdown of '%s' unsuccessful, using destroy",
                  instance.name)
Iustin Pop's avatar
Iustin Pop committed
963
964
965
966

    try:
      hyper.StopInstance(instance, force=True)
    except errors.HypervisorError, err:
967
968
969
      msg = "Failed to force stop instance %s: %s" % (instance.name, err)
      logging.error(msg)
      return (False, msg)
Iustin Pop's avatar
Iustin Pop committed
970
971

    time.sleep(1)
972
    if instance.name in GetInstanceList([hv_name]):
973
974
975
976
      msg = ("Could not shutdown instance %s even by destroy" %
             instance.name)
      logging.error(msg)
      return (False, msg)
Iustin Pop's avatar
Iustin Pop committed
977

978
  _RemoveBlockDevLinks(instance.name, instance.disks)
Iustin Pop's avatar
Iustin Pop committed
979

980
  return (True, "Instance has been shutdown successfully")
Iustin Pop's avatar
Iustin Pop committed
981
982


983
def InstanceReboot(instance, reboot_type):
984
985
  """Reboot an instance.

Iustin Pop's avatar
Iustin Pop committed
986
987
988
989
990
991
992
993
994
  @type instance: L{objects.Instance}
  @param instance: the instance object to reboot
  @type reboot_type: str
  @param reboot_type: the type of reboot, one the following
    constants:
      - L{constants.INSTANCE_REBOOT_SOFT}: only reboot the
        instance OS, do not recreate the VM
      - L{constants.INSTANCE_REBOOT_HARD}: tear down and
        restart the VM (at the hypervisor level)
995
996
997
998
      - the other reboot type (L{constants.INSTANCE_REBOOT_FULL}) is
        not accepted here, since that mode is handled differently, in
        cmdlib, and translates into full stop and start of the
        instance (instead of a call_instance_reboot RPC)
Iustin Pop's avatar
Iustin Pop committed
999
1000
  @rtype: boolean
  @return: the success of the operation
1001
1002

  """
1003
  running_instances = GetInstanceList([instance.hypervisor])
1004
1005

  if instance.name not in running_instances:
1006
1007
1008
    msg = "Cannot reboot instance %s that is not running" % instance.name
    logging.error(msg)
    return (False, msg)
1009

1010
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
1011
1012
1013
1014
  if reboot_type == constants.INSTANCE_REBOOT_SOFT:
    try:
      hyper.RebootInstance(instance)
    except errors.HypervisorError, err:
1015
1016
1017
      msg = "Failed to soft reboot instance %s: %s" % (instance.name, err)
      logging.error(msg)
      return (False, msg)
1018
1019
  elif reboot_type == constants.INSTANCE_REBOOT_HARD:
    try:
Iustin Pop's avatar
Iustin Pop committed
1020
1021
1022
      stop_result = InstanceShutdown(instance)
      if not stop_result[0]:
        return stop_result
1023
      return StartInstance(instance)
1024
    except errors.HypervisorError, err:
1025
1026
1027
      msg = "Failed to hard reboot instance %s: %s" % (instance.name, err)
      logging.error(msg)
      return (False, msg)
1028
  else:
1029
    return (False, "Invalid reboot_type received: %s" % (reboot_type,))
1030

1031
  return (True, "Reboot successful")
1032
1033


1034
1035
1036
1037
1038
1039
1040
def MigrationInfo(instance):
  """Gather information about an instance to be migrated.

  @type instance: L{objects.Instance}
  @param instance: the instance definition

  """
1041
1042
1043
1044
1045
1046
1047
1048
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
  try:
    info = hyper.MigrationInfo(instance)
  except errors.HypervisorError, err:
    msg = "Failed to fetch migration information"
    logging.exception(msg)
    return (False, '%s: %s' % (msg, err))
  return (True, info)
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061


def AcceptInstance(instance, info, target):
  """Prepare the node to accept an instance.

  @type instance: L{objects.Instance}
  @param instance: the instance definition
  @type info: string/data (opaque)
  @param info: migration information, from the source node
  @type target: string
  @param target: target host (usually ip), on this node

  """
1062
1063
1064
1065
1066
1067
1068
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
  try:
    hyper.AcceptInstance(instance, info, target)
  except errors.HypervisorError, err:
    msg = "Failed to accept instance"
    logging.exception(msg)
    return (False, '%s: %s' % (msg, err))
Michael Hanselmann's avatar
Michael Hanselmann committed
1069
  return (True, "Accept successful")
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082


def FinalizeMigration(instance, info, success):
  """Finalize any preparation to accept an instance.

  @type instance: L{objects.Instance}
  @param instance: the instance definition
  @type info: string/data (opaque)
  @param info: migration information, from the source node
  @type success: boolean
  @param success: whether the migration was a success or a failure

  """
1083
1084
1085
1086
1087
1088
1089
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
  try:
    hyper.FinalizeMigration(instance, info, success)
  except errors.HypervisorError, err:
    msg = "Failed to finalize migration"
    logging.exception(msg)
    return (False, '%s: %s' % (msg, err))
1090
1091
1092
  return (True, "Migration Finalized")


1093
1094
1095
def MigrateInstance(instance, target, live):
  """Migrates an instance to another node.

Iustin Pop's avatar
Iustin Pop committed
1096
  @type instance: L{objects.Instance}
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
  @param instance: the instance definition
  @type target: string
  @param target: the target node name
  @type live: boolean
  @param live: whether the migration should be done live or not (the
      interpretation of this parameter is left to the hypervisor)
  @rtype: tuple
  @return: a tuple of (success, msg) where:
      - succes is a boolean denoting the success/failure of the operation
      - msg is a string with details in case of failure

1108
  """
1109
  hyper = hypervisor.GetHypervisor(instance.hypervisor)
1110
1111

  try:
1112
    hyper.MigrateInstance(instance.name, target, live)
1113
  except errors.HypervisorError, err:
1114
1115
1116
    msg = "Failed to migrate instance"
    logging.exception(msg)
    return (False, "%s: %s" % (msg, err))
Michael Hanselmann's avatar
Michael Hanselmann committed
1117
  return (True, "Migration successful")
1118
1119


1120
def BlockdevCreate(disk, size, owner, on_primary, info):
Iustin Pop's avatar
Iustin Pop committed
1121
1122
  """Creates a block device for an instance.

Iustin Pop's avatar
Iustin Pop committed
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
  @type disk: L{objects.Disk}
  @param disk: the object describing the disk we should create
  @type size: int
  @param size: the size of the physical underlying device, in MiB
  @type owner: str
  @param owner: the name of the instance for which disk is created,
      used for device cache data
  @type on_primary: boolean
  @param on_primary:  indicates if it is the primary node or not
  @type info: string
  @param info: string that will be sent to the physical device
      creation, used for example to set (LVM) tags on LVs

  @return: the new unique_id of the device (this can sometime be
      computed only after creation), or None. On secondary nodes,
      it's not required to return anything.
Iustin Pop's avatar
Iustin Pop committed
1139
1140
1141
1142
1143

  """
  clist = []
  if disk.children:
    for child in disk.children:
1144
1145
1146
1147
1148
1149
      try:
        crdev = _RecursiveAssembleBD(child, owner, on_primary)
      except errors.BlockDeviceError, err:
        errmsg = "Can't assemble device %s: %s" % (child, err)
        logging.error(errmsg)
        return False, errmsg
Iustin Pop's avatar
Iustin Pop committed
1150
1151
1152
      if on_primary or disk.AssembleOnSecondary():
        # we need the children open in case the device itself has to
        # be assembled
1153
        try:
Iustin Pop's avatar
Iustin Pop committed
1154
          # pylint: disable-msg=E1103
1155
1156
          crdev.Open()
        except errors.BlockDeviceError, err:
Iustin Pop's avatar
Iustin Pop committed
1157
          errmsg = "Can't make child '%s' read-write: %s" % (child, err)
1158
1159
          logging.error(errmsg)
          return False, errmsg
Iustin Pop's avatar
Iustin Pop committed
1160
1161
      clist.append(crdev)

1162
  try:
Iustin Pop's avatar
Iustin Pop committed
1163
    device = bdev.Create(disk.dev_type, disk.physical_id, clist, disk.size)
1164
  except errors.BlockDeviceError, err:
1165
    return False, "Can't create block device: %s" % str(err)
Iustin Pop's avatar
Iustin Pop committed
1166

Iustin Pop's avatar
Iustin Pop committed
1167
  if on_primary or disk.AssembleOnSecondary():
1168
1169
1170
1171
1172
1173
1174
    try:
      device.Assemble()
    except errors.BlockDeviceError, err:
      errmsg = ("Can't assemble device after creation, very"
                " unusual event: %s" % str(err))
      logging.error(errmsg)
      return False, errmsg
1175
    device.SetSyncSpeed(constants.SYNC_SPEED)
Iustin Pop's avatar
Iustin Pop committed
1176
    if on_primary or disk.OpenOnSecondary():
1177
1178
1179
1180
1181
1182
1183
      try:
        device.Open(force=True)
      except errors.BlockDeviceError, err:
        errmsg = ("Can't make device r/w after creation, very"
                  " unusual event: %s" % str(err))
        logging.error(errmsg)
        ret