qa_node.py 16.8 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


Iustin Pop's avatar
Iustin Pop committed
22
23
24
25
"""Node-related QA tests.

"""

26
from ganeti import utils
27
from ganeti import constants
28
from ganeti import query
29
from ganeti import serializer
30
31
32

import qa_config
import qa_error
33
import qa_utils
34

35
from qa_utils import AssertCommand, AssertEqual, AssertIn, GetCommandOutput
36
37


38
def _NodeAdd(node, readd=False):
39
  if not readd and node.added:
40
    raise qa_error.Error("Node %s already in cluster" % node.primary)
41
  elif readd and not node.added:
42
    raise qa_error.Error("Node %s not yet in cluster" % node.primary)
Iustin Pop's avatar
Iustin Pop committed
43
44

  cmd = ["gnt-node", "add", "--no-ssh-key-check"]
45
46
  if node.secondary:
    cmd.append("--secondary-ip=%s" % node.secondary)
47
  if readd:
Iustin Pop's avatar
Iustin Pop committed
48
    cmd.append("--readd")
49
  cmd.append(node.primary)
Iustin Pop's avatar
Iustin Pop committed
50
51

  AssertCommand(cmd)
52

53
54
55
56
  if readd:
    assert node.added
  else:
    node.MarkAdded()
57
58
59


def _NodeRemove(node):
60
  AssertCommand(["gnt-node", "remove", node.primary])
61
  node.MarkRemoved()
62
63


64
65
66
def MakeNodeOffline(node, value):
  """gnt-node modify --offline=value"""
  # value in ["yes", "no"]
67
  AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
68
69


70
71
72
def TestNodeAddAll():
  """Adding all nodes to cluster."""
  master = qa_config.GetMasterNode()
Iustin Pop's avatar
Iustin Pop committed
73
  for node in qa_config.get("nodes"):
74
    if node != master:
75
      _NodeAdd(node, readd=False)
76
77


78
79
80
81
82
83
84
def MarkNodeAddedAll():
  """Mark all nodes as added.

  This is useful if we don't create the cluster ourselves (in qa).

  """
  master = qa_config.GetMasterNode()
Iustin Pop's avatar
Iustin Pop committed
85
  for node in qa_config.get("nodes"):
86
    if node != master:
87
      node.MarkAdded()
88
89


90
91
92
def TestNodeRemoveAll():
  """Removing all nodes from cluster."""
  master = qa_config.GetMasterNode()
Iustin Pop's avatar
Iustin Pop committed
93
  for node in qa_config.get("nodes"):
94
95
96
97
    if node != master:
      _NodeRemove(node)


98
99
100
101
102
def TestNodeReadd(node):
  """gnt-node add --readd"""
  _NodeAdd(node, readd=True)


103
104
def TestNodeInfo():
  """gnt-node info"""
Iustin Pop's avatar
Iustin Pop committed
105
  AssertCommand(["gnt-node", "info"])
106
107
108
109


def TestNodeVolumes():
  """gnt-node volumes"""
Iustin Pop's avatar
Iustin Pop committed
110
  AssertCommand(["gnt-node", "volumes"])
111
112


113
114
115
116
def TestNodeStorage():
  """gnt-node storage"""
  master = qa_config.GetMasterNode()

117
  # FIXME: test all storage_types in constants.STORAGE_TYPES
118
  # as soon as they are implemented.
119
120
121
  enabled_storage_types = qa_config.GetEnabledStorageTypes()
  testable_storage_types = list(set(enabled_storage_types).intersection(
      set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
Iustin Pop's avatar
Iustin Pop committed
122

123
  for storage_type in testable_storage_types:
Iustin Pop's avatar
Iustin Pop committed
124

125
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
Iustin Pop's avatar
Iustin Pop committed
126

127
    # Test simple list
Iustin Pop's avatar
Iustin Pop committed
128
    AssertCommand(cmd)
129
130
131
132
133

    # Test all storage fields
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
           "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
                                    [constants.SF_NODE, constants.SF_TYPE])]
Iustin Pop's avatar
Iustin Pop committed
134
    AssertCommand(cmd)
135
136
137
138
139

    # Get list of valid storage devices
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
           "--output=node,name,allocatable", "--separator=|",
           "--no-headers"]
140
    output = qa_utils.GetCommandOutput(master.primary,
141
142
143
144
145
146
147
148
149
150
                                       utils.ShellQuoteArgs(cmd))

    # Test with up to two devices
    testdevcount = 2

    for line in output.splitlines()[:testdevcount]:
      (node_name, st_name, st_allocatable) = line.split("|")

      # Dummy modification without any changes
      cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
Iustin Pop's avatar
Iustin Pop committed
151
      AssertCommand(cmd)
152
153
154
155
156
157
158

      # Make sure we end up with the same value as before
      if st_allocatable.lower() == "y":
        test_allocatable = ["no", "yes"]
      else:
        test_allocatable = ["yes", "no"]

Iustin Pop's avatar
Iustin Pop committed
159
160
      fail = (constants.SF_ALLOCATABLE not in
              constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
161
162

      for i in test_allocatable:
Iustin Pop's avatar
Iustin Pop committed
163
164
        AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
                       node_name, storage_type, st_name], fail=fail)
165

166
167
168
169
        # Verify list output
        cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
               "--output=name,allocatable", "--separator=|",
               "--no-headers", node_name]
170
        listout = qa_utils.GetCommandOutput(master.primary,
171
172
173
174
175
176
177
178
                                            utils.ShellQuoteArgs(cmd))
        for line in listout.splitlines():
          (vfy_name, vfy_allocatable) = line.split("|")
          if vfy_name == st_name and not fail:
            AssertEqual(vfy_allocatable, i[0].upper())
          else:
            AssertEqual(vfy_allocatable, st_allocatable)

179
      # Test repair functionality
Iustin Pop's avatar
Iustin Pop committed
180
181
182
183
      fail = (constants.SO_FIX_CONSISTENCY not in
              constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
      AssertCommand(["gnt-node", "repair-storage", node_name,
                     storage_type, st_name], fail=fail)
184
185


186
187
def TestNodeFailover(node, node2):
  """gnt-node failover"""
188
  if qa_utils.GetNodeInstances(node2, secondaries=False):
189
190
191
    raise qa_error.UnusableNodeError("Secondary node has at least one"
                                     " primary instance. This test requires"
                                     " it to have no primary instances.")
192
193

  # Fail over to secondary node
194
  AssertCommand(["gnt-node", "failover", "-f", node.primary])
195
196

  # ... and back again.
197
  AssertCommand(["gnt-node", "failover", "-f", node2.primary])
198
199


Thomas Thrainer's avatar
Thomas Thrainer committed
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def TestNodeMigrate(node, node2):
  """gnt-node migrate"""
  if qa_utils.GetNodeInstances(node2, secondaries=False):
    raise qa_error.UnusableNodeError("Secondary node has at least one"
                                     " primary instance. This test requires"
                                     " it to have no primary instances.")

  # Migrate to secondary node
  AssertCommand(["gnt-node", "migrate", "-f", node.primary])

  # ... and back again.
  AssertCommand(["gnt-node", "migrate", "-f", node2.primary])


214
215
216
217
def TestNodeEvacuate(node, node2):
  """gnt-node evacuate"""
  node3 = qa_config.AcquireNode(exclude=[node, node2])
  try:
218
    if qa_utils.GetNodeInstances(node3, secondaries=True):
219
220
221
      raise qa_error.UnusableNodeError("Evacuation node has at least one"
                                       " secondary instance. This test requires"
                                       " it to have no secondary instances.")
222
223

    # Evacuate all secondary instances
Iustin Pop's avatar
Iustin Pop committed
224
    AssertCommand(["gnt-node", "evacuate", "-f",
225
                   "--new-secondary=%s" % node3.primary, node2.primary])
226
227

    # ... and back again.
Iustin Pop's avatar
Iustin Pop committed
228
    AssertCommand(["gnt-node", "evacuate", "-f",
229
                   "--new-secondary=%s" % node2.primary, node3.primary])
230
  finally:
231
    node3.Release()
232
233
234
235


def TestNodeModify(node):
  """gnt-node modify"""
236
237
238
239
240
241
242

  # make sure enough master candidates will be available by disabling the
  # master candidate role first with --auto-promote
  AssertCommand(["gnt-node", "modify", "--master-candidate=no",
                "--auto-promote", node.primary])

  # now it's save to force-remove the master candidate role
243
244
  for flag in ["master-candidate", "drained", "offline"]:
    for value in ["yes", "no"]:
Iustin Pop's avatar
Iustin Pop committed
245
      AssertCommand(["gnt-node", "modify", "--force",
246
                     "--%s=%s" % (flag, value), node.primary])
Iustin Pop's avatar
Iustin Pop committed
247

248
  AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
249

250
  # Test setting secondary IP address
251
252
  AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
                 node.primary])
253

254

255
256
257
258
def _CreateOobScriptStructure():
  """Create a simple OOB handling script and its structure."""
  master = qa_config.GetMasterNode()

259
260
261
  data_path = qa_utils.UploadData(master.primary, "")
  verify_path = qa_utils.UploadData(master.primary, "")
  exit_code_path = qa_utils.UploadData(master.primary, "")
262
263
264
265
266
267
268

  oob_script = (("#!/bin/bash\n"
                 "echo \"$@\" > %s\n"
                 "cat %s\n"
                 "exit $(< %s)\n") %
                (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
                 utils.ShellQuote(exit_code_path)))
269
  oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
270
271
272
273
274
275
276

  return [oob_path, verify_path, data_path, exit_code_path]


def _UpdateOobFile(path, data):
  """Updates the data file with data."""
  master = qa_config.GetMasterNode()
277
  qa_utils.UploadData(master.primary, data, filename=path)
278
279
280
281
282
283
284


def _AssertOobCall(verify_path, expected_args):
  """Assert the OOB call was performed with expetected args."""
  master = qa_config.GetMasterNode()

  verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
285
  output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
286
                                     tty=False)
287

288
  AssertEqual(expected_args, output.strip())
289
290
291
292
293
294


def TestOutOfBand():
  """gnt-node power"""
  master = qa_config.GetMasterNode()

Iustin Pop's avatar
Iustin Pop committed
295
  node = qa_config.AcquireNode(exclude=master)
Michael Hanselmann's avatar
Michael Hanselmann committed
296

297
298
  master_name = master.primary
  node_name = node.primary
Michael Hanselmann's avatar
Michael Hanselmann committed
299
  full_node_name = qa_utils.ResolveNodeName(node)
300

301
302
303
304
305
306
307
308
309
310
  (oob_path, verify_path,
   data_path, exit_code_path) = _CreateOobScriptStructure()

  try:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
                   "oob_program=%s" % oob_path])

    # No data, exit 0
    _UpdateOobFile(exit_code_path, "0")

311
    AssertCommand(["gnt-node", "power", "on", node_name])
Michael Hanselmann's avatar
Michael Hanselmann committed
312
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
313

314
    AssertCommand(["gnt-node", "power", "-f", "off", node_name])
Michael Hanselmann's avatar
Michael Hanselmann committed
315
    _AssertOobCall(verify_path, "power-off %s" % full_node_name)
316

317
    # Power off on master without options should fail
318
    AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
319
    # With force master it should still fail
Andrea Spadaccini's avatar
Andrea Spadaccini committed
320
    AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
321
                   master_name],
322
323
                  fail=True)

324
325
326
327
328
329
330
    # Verify we can't transform back to online when not yet powered on
    AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
                  fail=True)
    # Now reset state
    AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
                   node_name])

331
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
Michael Hanselmann's avatar
Michael Hanselmann committed
332
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
333

René Nussbaumer's avatar
René Nussbaumer committed
334
335
    # Those commands should fail as they expect output which isn't provided yet
    # But they should have called the oob helper nevermind
336
    AssertCommand(["gnt-node", "power", "status", node_name],
337
                  fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
338
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
339

René Nussbaumer's avatar
René Nussbaumer committed
340
341
342
343
344
345
346
    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

    # Correct Data, exit 0
Andrea Spadaccini's avatar
Andrea Spadaccini committed
347
    _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
348

349
    AssertCommand(["gnt-node", "power", "status", node_name])
Michael Hanselmann's avatar
Michael Hanselmann committed
350
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
351

René Nussbaumer's avatar
René Nussbaumer committed
352
353
354
355
356
357
358
359
360
    _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
                                                   ["disk0", "CRITICAL"]]))

    AssertCommand(["gnt-node", "health", node_name])
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"])

    # Those commands should fail as they expect no data regardless of exit 0
361
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
362
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
363

364
    try:
365
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
366
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
367
368
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
369

370
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
371
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
372
373
374
375

    # Data, exit 1 (all should fail)
    _UpdateOobFile(exit_code_path, "1")

376
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
377
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
378

379
    try:
380
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
381
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
382
383
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
384

385
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
386
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
387

388
    AssertCommand(["gnt-node", "power", "status", node_name],
389
                  fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
390
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
391

René Nussbaumer's avatar
René Nussbaumer committed
392
393
394
395
396
397
    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

398
399
    # No data, exit 1 (all should fail)
    _UpdateOobFile(data_path, "")
400
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
401
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
402

403
    try:
404
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
405
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
406
407
    finally:
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
408

409
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
410
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
411

412
    AssertCommand(["gnt-node", "power", "status", node_name],
413
                  fail=True)
Michael Hanselmann's avatar
Michael Hanselmann committed
414
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
415

René Nussbaumer's avatar
René Nussbaumer committed
416
417
418
419
420
421
    AssertCommand(["gnt-node", "health", node_name],
                  fail=True)
    _AssertOobCall(verify_path, "health %s" % full_node_name)

    AssertCommand(["gnt-node", "health"], fail=True)

422
    # Different OOB script for node
423
    verify_path2 = qa_utils.UploadData(master.primary, "")
424
425
    oob_script = ("#!/bin/sh\n"
                  "echo \"$@\" > %s\n") % verify_path2
426
    oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
427
428
429

    try:
      AssertCommand(["gnt-node", "modify", "--node-parameters",
430
431
                     "oob_program=%s" % oob_path2, node_name])
      AssertCommand(["gnt-node", "power", "on", node_name])
Michael Hanselmann's avatar
Michael Hanselmann committed
432
      _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
433
434
    finally:
      AssertCommand(["gnt-node", "modify", "--node-parameters",
435
                     "oob_program=default", node_name])
436
437
438
      AssertCommand(["rm", "-f", oob_path2, verify_path2])
  finally:
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
439
                   "oob_program="])
440
441
442
443
    AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
                   exit_code_path])


444
445
446
def TestNodeList():
  """gnt-node list"""
  qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
447
448
449
450
451


def TestNodeListFields():
  """gnt-node list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
452
453


454
def TestNodeListDrbd(node, is_drbd):
455
  """gnt-node list-drbd"""
456
457
458
459
460
461
462
463
  master = qa_config.GetMasterNode()
  result_output = GetCommandOutput(master.primary,
                                   "gnt-node list-drbd --no-header %s" %
                                   node.primary)
  # Meaningful to note: there is but one instance, and the node is either the
  # primary or one of the secondaries
  if is_drbd:
    # Invoked for both primary and secondary
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
464
465
466
467
468
469
470
471
472
473
474
    per_disk_info = result_output.splitlines()
    for line in per_disk_info:
      try:
        drbd_node, _, _, _, _, drbd_peer = line.split()
      except ValueError:
        raise qa_error.Error("Could not examine list-drbd output: expected a"
                             " single row of 6 entries, found the following:"
                             " %s" % line)

      AssertIn(node.primary, [drbd_node, drbd_peer],
               msg="The output %s does not contain the node" % line)
475
476
477
  else:
    # Output should be empty, barring newlines
    AssertEqual(result_output.strip(), "")
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495


def _BuildSetESCmd(action, value, node_name):
  cmd = ["gnt-node"]
  if action == "add":
    cmd.extend(["add", "--readd"])
  else:
    cmd.append("modify")
  cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
  return cmd


def TestExclStorSingleNode(node):
  """gnt-node add/modify cannot change the exclusive_storage flag.

  """
  for action in ["add", "modify"]:
    for value in (True, False, "default"):
Michael Hanselmann's avatar
Michael Hanselmann committed
496
      AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)