cfgupgrade 19.4 KB
Newer Older
1
2
3
#!/usr/bin/python
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Tool to upgrade the configuration file.

24
25
This code handles only the types supported by simplejson. As an
example, 'set' is a 'list'.
26
27
28
29
30
31
32
33

"""


import os
import os.path
import sys
import optparse
34
import logging
35
36
import time
from cStringIO import StringIO
37

38
39
from ganeti import constants
from ganeti import serializer
40
from ganeti import utils
41
from ganeti import cli
42
from ganeti import bootstrap
43
from ganeti import config
44
from ganeti import netutils
45
from ganeti import pathutils
46
47


48
49
options = None
args = None
50

Iustin Pop's avatar
Iustin Pop committed
51

52
53
54
#: Target major version we will upgrade to
TARGET_MAJOR = 2
#: Target minor version we will upgrade to
Guido Trotter's avatar
Guido Trotter committed
55
TARGET_MINOR = 7
56
57
58
59
#: Target major version for downgrade
DOWNGRADE_MAJOR = 2
#: Target minor version for downgrade
DOWNGRADE_MINOR = 7
60
61


62
63
64
class Error(Exception):
  """Generic exception"""
  pass
65
66


67
68
69
70
71
72
73
74
75
76
77
78
79
def SetupLogging():
  """Configures the logging module.

  """
  formatter = logging.Formatter("%(asctime)s: %(message)s")

  stderr_handler = logging.StreamHandler()
  stderr_handler.setFormatter(formatter)
  if options.debug:
    stderr_handler.setLevel(logging.NOTSET)
  elif options.verbose:
    stderr_handler.setLevel(logging.INFO)
  else:
80
    stderr_handler.setLevel(logging.WARNING)
81
82
83
84
85
86

  root_logger = logging.getLogger("")
  root_logger.setLevel(logging.NOTSET)
  root_logger.addHandler(stderr_handler)


87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
def CheckHostname(path):
  """Ensures hostname matches ssconf value.

  @param path: Path to ssconf file

  """
  ssconf_master_node = utils.ReadOneLineFile(path)
  hostname = netutils.GetHostname().name

  if ssconf_master_node == hostname:
    return True

  logging.warning("Warning: ssconf says master node is '%s', but this"
                  " machine's name is '%s'; this tool must be run on"
                  " the master node", ssconf_master_node, hostname)
  return False

104

105
106
def _FillIPolicySpecs(default_ipolicy, ipolicy):
  if "minmax" in ipolicy:
107
108
    for (key, spec) in ipolicy["minmax"][0].items():
      for (par, val) in default_ipolicy["minmax"][0][key].items():
109
110
111
112
113
        if par not in spec:
          spec[par] = val


def UpgradeIPolicy(ipolicy, default_ipolicy, isgroup):
114
115
116
117
118
  minmax_keys = ["min", "max"]
  if any((k in ipolicy) for k in minmax_keys):
    minmax = {}
    for key in minmax_keys:
      if key in ipolicy:
119
120
        if ipolicy[key]:
          minmax[key] = ipolicy[key]
121
        del ipolicy[key]
122
    if minmax:
123
      ipolicy["minmax"] = [minmax]
124
125
126
  if isgroup and "std" in ipolicy:
    del ipolicy["std"]
  _FillIPolicySpecs(default_ipolicy, ipolicy)
127
128


Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
129
130
131
132
133
134
def UpgradeNetworks(config_data):
  networks = config_data.get("networks", None)
  if not networks:
    config_data["networks"] = {}


135
136
137
138
def UpgradeCluster(config_data):
  cluster = config_data.get("cluster", None)
  if cluster is None:
    raise Error("Cannot find cluster")
139
  ipolicy = cluster.setdefault("ipolicy", None)
140
  if ipolicy:
141
    UpgradeIPolicy(ipolicy, constants.IPOLICY_DEFAULTS, False)
142
143


Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
144
def UpgradeGroups(config_data):
145
  cl_ipolicy = config_data["cluster"].get("ipolicy")
Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
146
147
148
149
  for group in config_data["nodegroups"].values():
    networks = group.get("networks", None)
    if not networks:
      group["networks"] = {}
150
151
    ipolicy = group.get("ipolicy", None)
    if ipolicy:
152
153
154
155
      if cl_ipolicy is None:
        raise Error("A group defines an instance policy but there is no"
                    " instance policy at cluster level")
      UpgradeIPolicy(ipolicy, cl_ipolicy, True)
Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
156

157

158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
def GetExclusiveStorageValue(config_data):
  """Return a conservative value of the exclusive_storage flag.

  Return C{True} if the cluster or at least a nodegroup have the flag set.

  """
  ret = False
  cluster = config_data["cluster"]
  ndparams = cluster.get("ndparams")
  if ndparams is not None and ndparams.get("exclusive_storage"):
    ret = True
  for group in config_data["nodegroups"].values():
    ndparams = group.get("ndparams")
    if ndparams is not None and ndparams.get("exclusive_storage"):
      ret = True
  return ret


176
177
178
def UpgradeInstances(config_data):
  network2uuid = dict((n["name"], n["uuid"])
                      for n in config_data["networks"].values())
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
179
180
181
  if "instances" not in config_data:
    raise Error("Can't find the 'instances' key in the configuration!")

182
  missing_spindles = False
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
183
184
  for instance, iobj in config_data["instances"].items():
    for nic in iobj["nics"]:
185
186
187
188
189
190
191
192
      name = nic.get("network", None)
      if name:
        uuid = network2uuid.get(name, None)
        if uuid:
          print("NIC with network name %s found."
                " Substituting with uuid %s." % (name, uuid))
          nic["network"] = uuid

Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
193
194
195
196
197
198
199
200
201
202
203
    if "disks" not in iobj:
      raise Error("Instance '%s' doesn't have a disks entry?!" % instance)
    disks = iobj["disks"]
    for idx, dobj in enumerate(disks):
      expected = "disk/%s" % idx
      current = dobj.get("iv_name", "")
      if current != expected:
        logging.warning("Updating iv_name for instance %s/disk %s"
                        " from '%s' to '%s'",
                        instance, idx, current, expected)
        dobj["iv_name"] = expected
204
205
206
207
208
209
210
211
212
213
214
      if not "spindles" in dobj:
        missing_spindles = True

  if GetExclusiveStorageValue(config_data) and missing_spindles:
    # We cannot be sure that the instances that are missing spindles have
    # exclusive storage enabled (the check would be more complicated), so we
    # give a noncommittal message
    logging.warning("Some instance disks could be needing to update the"
                    " spindles parameter; you can check by running"
                    " 'gnt-cluster verify', and fix any problem with"
                    " 'gnt-cluster repair-disk-sizes'")
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280


def UpgradeRapiUsers():
  if (os.path.isfile(options.RAPI_USERS_FILE_PRE24) and
      not os.path.islink(options.RAPI_USERS_FILE_PRE24)):
    if os.path.exists(options.RAPI_USERS_FILE):
      raise Error("Found pre-2.4 RAPI users file at %s, but another file"
                  " already exists at %s" %
                  (options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE))
    logging.info("Found pre-2.4 RAPI users file at %s, renaming to %s",
                 options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE)
    if not options.dry_run:
      utils.RenameFile(options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE,
                       mkdir=True, mkdir_mode=0750)

  # Create a symlink for RAPI users file
  if (not (os.path.islink(options.RAPI_USERS_FILE_PRE24) or
           os.path.isfile(options.RAPI_USERS_FILE_PRE24)) and
      os.path.isfile(options.RAPI_USERS_FILE)):
    logging.info("Creating symlink from %s to %s",
                 options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE)
    if not options.dry_run:
      os.symlink(options.RAPI_USERS_FILE, options.RAPI_USERS_FILE_PRE24)


def UpgradeWatcher():
  # Remove old watcher state file if it exists
  if os.path.exists(options.WATCHER_STATEFILE):
    logging.info("Removing watcher state file %s", options.WATCHER_STATEFILE)
    if not options.dry_run:
      utils.RemoveFile(options.WATCHER_STATEFILE)


def UpgradeFileStoragePaths(config_data):
  # Write file storage paths
  if not os.path.exists(options.FILE_STORAGE_PATHS_FILE):
    cluster = config_data["cluster"]
    file_storage_dir = cluster.get("file_storage_dir")
    shared_file_storage_dir = cluster.get("shared_file_storage_dir")
    del cluster

    logging.info("Ganeti 2.7 and later only allow whitelisted directories"
                 " for file storage; writing existing configuration values"
                 " into '%s'",
                 options.FILE_STORAGE_PATHS_FILE)

    if file_storage_dir:
      logging.info("File storage directory: %s", file_storage_dir)
    if shared_file_storage_dir:
      logging.info("Shared file storage directory: %s",
                   shared_file_storage_dir)

    buf = StringIO()
    buf.write("# List automatically generated from configuration by\n")
    buf.write("# cfgupgrade at %s\n" % time.asctime())
    if file_storage_dir:
      buf.write("%s\n" % file_storage_dir)
    if shared_file_storage_dir:
      buf.write("%s\n" % shared_file_storage_dir)
    utils.WriteFile(file_name=options.FILE_STORAGE_PATHS_FILE,
                    data=buf.getvalue(),
                    mode=0600,
                    dry_run=options.dry_run,
                    backup=True)


281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
def GetNewNodeIndex(nodes_by_old_key, old_key, new_key_field):
  if old_key not in nodes_by_old_key:
    logging.warning("Can't find node '%s' in configuration, assuming that it's"
                    " already up-to-date", old_key)
    return old_key
  return nodes_by_old_key[old_key][new_key_field]


def ChangeNodeIndices(config_data, old_key_field, new_key_field):
  def ChangeDiskNodeIndices(disk):
    if disk["dev_type"] in constants.LDS_DRBD:
      for i in range(0, 2):
        disk["logical_id"][i] = GetNewNodeIndex(nodes_by_old_key,
                                                disk["logical_id"][i],
                                                new_key_field)
    if "children" in disk:
      for child in disk["children"]:
        ChangeDiskNodeIndices(child)

  nodes_by_old_key = {}
  nodes_by_new_key = {}
  for (_, node) in config_data["nodes"].items():
    nodes_by_old_key[node[old_key_field]] = node
    nodes_by_new_key[node[new_key_field]] = node

  config_data["nodes"] = nodes_by_new_key

  cluster = config_data["cluster"]
  cluster["master_node"] = GetNewNodeIndex(nodes_by_old_key,
                                           cluster["master_node"],
                                           new_key_field)

  for inst in config_data["instances"].values():
    inst["primary_node"] = GetNewNodeIndex(nodes_by_old_key,
                                           inst["primary_node"],
                                           new_key_field)
    for disk in inst["disks"]:
      ChangeDiskNodeIndices(disk)


def UpgradeNodeIndices(config_data):
  ChangeNodeIndices(config_data, "name", "uuid")


Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
325
326
327
328
329
330
331
def UpgradeAll(config_data):
  config_data["version"] = constants.BuildVersion(TARGET_MAJOR,
                                                  TARGET_MINOR, 0)
  UpgradeRapiUsers()
  UpgradeWatcher()
  UpgradeFileStoragePaths(config_data)
  UpgradeNetworks(config_data)
332
  UpgradeCluster(config_data)
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
333
334
  UpgradeGroups(config_data)
  UpgradeInstances(config_data)
335
  UpgradeNodeIndices(config_data)
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
336

337

338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
def DowngradeDisks(disks, owner):
  for disk in disks:
    # Remove spindles to downgrade to 2.8
    if "spindles" in disk:
      logging.warning("Removing spindles (value=%s) from disk %s (%s) of"
                      " instance %s",
                      disk["spindles"], disk["iv_name"], disk["uuid"], owner)
      del disk["spindles"]


def DowngradeInstances(config_data):
  if "instances" not in config_data:
    raise Error("Cannot find the 'instances' key in the configuration!")
  for (iname, iobj) in config_data["instances"].items():
    if "disks" not in iobj:
      raise Error("Cannot find 'disks' key for instance %s" % iname)
    DowngradeDisks(iobj["disks"], iname)


357
358
359
360
def DowngradeNodeIndices(config_data):
  ChangeNodeIndices(config_data, "uuid", "name")


361
362
363
def DowngradeAll(config_data):
  # Any code specific to a particular version should be labeled that way, so
  # it can be removed when updating to the next version.
364
  DowngradeInstances(config_data)
365
  DowngradeNodeIndices(config_data)
366
367


368
369
370
371
def main():
  """Main program.

  """
372
  global options, args # pylint: disable=W0603
373

374
  # Option parsing
375
  parser = optparse.OptionParser(usage="%prog [--debug|--verbose] [--force]")
376
  parser.add_option("--dry-run", dest="dry_run",
377
                    action="store_true",
378
379
                    help="Try to do the conversion, but don't write"
                         " output file")
380
  parser.add_option(cli.FORCE_OPT)
381
  parser.add_option(cli.DEBUG_OPT)
382
  parser.add_option(cli.VERBOSE_OPT)
383
384
385
  parser.add_option("--ignore-hostname", dest="ignore_hostname",
                    action="store_true", default=False,
                    help="Don't abort if hostname doesn't match")
386
  parser.add_option("--path", help="Convert configuration in this"
387
388
                    " directory instead of '%s'" % pathutils.DATA_DIR,
                    default=pathutils.DATA_DIR, dest="data_dir")
389
390
391
392
  parser.add_option("--confdir",
                    help=("Use this directory instead of '%s'" %
                          pathutils.CONF_DIR),
                    default=pathutils.CONF_DIR, dest="conf_dir")
393
394
395
  parser.add_option("--no-verify",
                    help="Do not verify configuration after upgrade",
                    action="store_true", dest="no_verify", default=False)
396
397
398
  parser.add_option("--downgrade",
                    help="Downgrade to the previous stable version",
                    action="store_true", dest="downgrade", default=False)
399
400
  (options, args) = parser.parse_args()

401
402
  # We need to keep filenames locally because they might be renamed between
  # versions.
403
  options.data_dir = os.path.abspath(options.data_dir)
404
405
406
407
  options.CONFIG_DATA_PATH = options.data_dir + "/config.data"
  options.SERVER_PEM_PATH = options.data_dir + "/server.pem"
  options.KNOWN_HOSTS_PATH = options.data_dir + "/known_hosts"
  options.RAPI_CERT_FILE = options.data_dir + "/rapi.pem"
408
409
  options.SPICE_CERT_FILE = options.data_dir + "/spice.pem"
  options.SPICE_CACERT_FILE = options.data_dir + "/spice-ca.pem"
410
411
  options.RAPI_USERS_FILE = options.data_dir + "/rapi/users"
  options.RAPI_USERS_FILE_PRE24 = options.data_dir + "/rapi_users"
412
  options.CONFD_HMAC_KEY = options.data_dir + "/hmac.key"
413
  options.CDS_FILE = options.data_dir + "/cluster-domain-secret"
414
  options.SSCONF_MASTER_NODE = options.data_dir + "/ssconf_master_node"
415
  options.WATCHER_STATEFILE = options.data_dir + "/watcher.data"
416
  options.FILE_STORAGE_PATHS_FILE = options.conf_dir + "/file-storage-paths"
417

418
419
  SetupLogging()

420
421
  # Option checking
  if args:
422
    raise Error("No arguments expected")
423
424
  if options.downgrade and not options.no_verify:
    options.no_verify = True
425

426
427
428
429
430
  # Check master name
  if not (CheckHostname(options.SSCONF_MASTER_NODE) or options.ignore_hostname):
    logging.error("Aborting due to hostname mismatch")
    sys.exit(constants.EXIT_FAILURE)

431
  if not options.force:
432
433
434
435
436
437
438
439
440
441
442
443
444
    if options.downgrade:
      usertext = ("The configuration is going to be DOWNGRADED to version %s.%s"
                  " Some configuration data might be removed if they don't fit"
                  " in the old format. Please make sure you have read the"
                  " upgrade notes (available in the UPGRADE file and included"
                  " in other documentation formats) to understand what they"
                  " are. Continue with *DOWNGRADING* the configuration?" %
                  (DOWNGRADE_MAJOR, DOWNGRADE_MINOR))
    else:
      usertext = ("Please make sure you have read the upgrade notes for"
                  " Ganeti %s (available in the UPGRADE file and included"
                  " in other documentation formats). Continue with upgrading"
                  " configuration?" % constants.RELEASE_VERSION)
445
    if not cli.AskUser(usertext):
446
      sys.exit(constants.EXIT_FAILURE)
447

448
  # Check whether it's a Ganeti configuration directory
449
  if not (os.path.isfile(options.CONFIG_DATA_PATH) and
450
          os.path.isfile(options.SERVER_PEM_PATH) and
451
          os.path.isfile(options.KNOWN_HOSTS_PATH)):
452
    raise Error(("%s does not seem to be a Ganeti configuration"
453
                 " directory") % options.data_dir)
454

455
456
457
  if not os.path.isdir(options.conf_dir):
    raise Error("Not a directory: %s" % options.conf_dir)

458
  config_data = serializer.LoadJson(utils.ReadFile(options.CONFIG_DATA_PATH))
459

460
461
462
463
  try:
    config_version = config_data["version"]
  except KeyError:
    raise Error("Unable to determine configuration version")
464

465
466
  (config_major, config_minor, config_revision) = \
    constants.SplitVersion(config_version)
467

468
469
  logging.info("Found configuration version %s (%d.%d.%d)",
               config_version, config_major, config_minor, config_revision)
470

471
472
473
  if "config_version" in config_data["cluster"]:
    raise Error("Inconsistent configuration: found config_version in"
                " configuration file")
474

475
476
477
478
479
480
481
482
483
  # Downgrade to the previous stable version
  if options.downgrade:
    if config_major != TARGET_MAJOR or config_minor != TARGET_MINOR:
      raise Error("Downgrade supported only from the latest version (%s.%s),"
                  " found %s (%s.%s.%s) instead" %
                  (TARGET_MAJOR, TARGET_MINOR, config_version, config_major,
                   config_minor, config_revision))
    DowngradeAll(config_data)

484
485
  # Upgrade from 2.{0..7} to 2.7
  elif config_major == 2 and config_minor in range(0, 8):
486
    if config_revision != 0:
487
      logging.warning("Config revision is %s, not 0", config_revision)
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
488
    UpgradeAll(config_data)
489

490
  elif config_major == TARGET_MAJOR and config_minor == TARGET_MINOR:
491
492
493
494
495
    logging.info("No changes necessary")

  else:
    raise Error("Configuration version %d.%d.%d not supported by this tool" %
                (config_major, config_minor, config_revision))
496

497
498
499
500
501
502
503
  try:
    logging.info("Writing configuration file to %s", options.CONFIG_DATA_PATH)
    utils.WriteFile(file_name=options.CONFIG_DATA_PATH,
                    data=serializer.DumpJson(config_data),
                    mode=0600,
                    dry_run=options.dry_run,
                    backup=True)
504
505

    if not options.dry_run:
Iustin Pop's avatar
Iustin Pop committed
506
507
508
509
510
511
512
513
      bootstrap.GenerateClusterCrypto(
        False, False, False, False, False,
        nodecert_file=options.SERVER_PEM_PATH,
        rapicert_file=options.RAPI_CERT_FILE,
        spicecert_file=options.SPICE_CERT_FILE,
        spicecacert_file=options.SPICE_CACERT_FILE,
        hmackey_file=options.CONFD_HMAC_KEY,
        cds_file=options.CDS_FILE)
514

515
  except Exception:
516
    logging.critical("Writing configuration failed. It is probably in an"
517
518
                     " inconsistent state and needs manual intervention.")
    raise
519

520
  # test loading the config file
521
  all_ok = True
522
  if not (options.dry_run or options.no_verify):
523
524
    logging.info("Testing the new config file...")
    cfg = config.ConfigWriter(cfg_file=options.CONFIG_DATA_PATH,
525
                              accept_foreign=options.ignore_hostname,
526
527
528
529
530
531
                              offline=True)
    # if we reached this, it's all fine
    vrfy = cfg.VerifyConfig()
    if vrfy:
      logging.error("Errors after conversion:")
      for item in vrfy:
Iustin Pop's avatar
Iustin Pop committed
532
        logging.error(" - %s", item)
533
534
535
      all_ok = False
    else:
      logging.info("File loaded successfully after upgrading")
536
537
    del cfg

538
539
540
541
542
543
  if options.downgrade:
    action = "downgraded"
    out_ver = "%s.%s" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR)
  else:
    action = "upgraded"
    out_ver = constants.RELEASE_VERSION
544
  if all_ok:
545
546
    cli.ToStderr("Configuration successfully %s to version %s.",
                 action, out_ver)
547
  else:
548
549
    cli.ToStderr("Configuration %s to version %s, but there are errors."
                 "\nPlease review the file.", action, out_ver)
550

551
552
553

if __name__ == "__main__":
  main()