cfgupgrade 17.6 KB
Newer Older
1
2
3
#!/usr/bin/python
#

4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Tool to upgrade the configuration file.

24
25
This code handles only the types supported by simplejson. As an
example, 'set' is a 'list'.
26
27
28
29
30
31
32
33

"""


import os
import os.path
import sys
import optparse
34
import logging
35
36
import time
from cStringIO import StringIO
37

38
39
from ganeti import constants
from ganeti import serializer
40
from ganeti import utils
41
from ganeti import cli
42
from ganeti import bootstrap
43
from ganeti import config
44
from ganeti import netutils
45
from ganeti import pathutils
46
47


48
49
options = None
args = None
50

Iustin Pop's avatar
Iustin Pop committed
51

52
53
54
#: Target major version we will upgrade to
TARGET_MAJOR = 2
#: Target minor version we will upgrade to
Guido Trotter's avatar
Guido Trotter committed
55
TARGET_MINOR = 7
56
57
58
59
#: Target major version for downgrade
DOWNGRADE_MAJOR = 2
#: Target minor version for downgrade
DOWNGRADE_MINOR = 7
60
61


62
63
64
class Error(Exception):
  """Generic exception"""
  pass
65
66


67
68
69
70
71
72
73
74
75
76
77
78
79
def SetupLogging():
  """Configures the logging module.

  """
  formatter = logging.Formatter("%(asctime)s: %(message)s")

  stderr_handler = logging.StreamHandler()
  stderr_handler.setFormatter(formatter)
  if options.debug:
    stderr_handler.setLevel(logging.NOTSET)
  elif options.verbose:
    stderr_handler.setLevel(logging.INFO)
  else:
80
    stderr_handler.setLevel(logging.WARNING)
81
82
83
84
85
86

  root_logger = logging.getLogger("")
  root_logger.setLevel(logging.NOTSET)
  root_logger.addHandler(stderr_handler)


87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
def CheckHostname(path):
  """Ensures hostname matches ssconf value.

  @param path: Path to ssconf file

  """
  ssconf_master_node = utils.ReadOneLineFile(path)
  hostname = netutils.GetHostname().name

  if ssconf_master_node == hostname:
    return True

  logging.warning("Warning: ssconf says master node is '%s', but this"
                  " machine's name is '%s'; this tool must be run on"
                  " the master node", ssconf_master_node, hostname)
  return False

104

105
106
def _FillIPolicySpecs(default_ipolicy, ipolicy):
  if "minmax" in ipolicy:
107
108
    for (key, spec) in ipolicy["minmax"][0].items():
      for (par, val) in default_ipolicy["minmax"][0][key].items():
109
110
111
112
113
        if par not in spec:
          spec[par] = val


def UpgradeIPolicy(ipolicy, default_ipolicy, isgroup):
114
115
116
117
118
  minmax_keys = ["min", "max"]
  if any((k in ipolicy) for k in minmax_keys):
    minmax = {}
    for key in minmax_keys:
      if key in ipolicy:
119
120
        if ipolicy[key]:
          minmax[key] = ipolicy[key]
121
        del ipolicy[key]
122
    if minmax:
123
      ipolicy["minmax"] = [minmax]
124
125
126
  if isgroup and "std" in ipolicy:
    del ipolicy["std"]
  _FillIPolicySpecs(default_ipolicy, ipolicy)
127
128


Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
129
130
131
132
133
134
def UpgradeNetworks(config_data):
  networks = config_data.get("networks", None)
  if not networks:
    config_data["networks"] = {}


135
136
137
138
def UpgradeCluster(config_data):
  cluster = config_data.get("cluster", None)
  if cluster is None:
    raise Error("Cannot find cluster")
139
  ipolicy = cluster.setdefault("ipolicy", None)
140
  if ipolicy:
141
    UpgradeIPolicy(ipolicy, constants.IPOLICY_DEFAULTS, False)
142
143


Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
144
def UpgradeGroups(config_data):
145
  cl_ipolicy = config_data["cluster"].get("ipolicy")
Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
146
147
148
149
  for group in config_data["nodegroups"].values():
    networks = group.get("networks", None)
    if not networks:
      group["networks"] = {}
150
151
    ipolicy = group.get("ipolicy", None)
    if ipolicy:
152
153
154
155
      if cl_ipolicy is None:
        raise Error("A group defines an instance policy but there is no"
                    " instance policy at cluster level")
      UpgradeIPolicy(ipolicy, cl_ipolicy, True)
Dimitris Aragiorgis's avatar
Dimitris Aragiorgis committed
156

157

158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
def GetExclusiveStorageValue(config_data):
  """Return a conservative value of the exclusive_storage flag.

  Return C{True} if the cluster or at least a nodegroup have the flag set.

  """
  ret = False
  cluster = config_data["cluster"]
  ndparams = cluster.get("ndparams")
  if ndparams is not None and ndparams.get("exclusive_storage"):
    ret = True
  for group in config_data["nodegroups"].values():
    ndparams = group.get("ndparams")
    if ndparams is not None and ndparams.get("exclusive_storage"):
      ret = True
  return ret


176
177
178
def UpgradeInstances(config_data):
  network2uuid = dict((n["name"], n["uuid"])
                      for n in config_data["networks"].values())
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
179
180
181
  if "instances" not in config_data:
    raise Error("Can't find the 'instances' key in the configuration!")

182
  missing_spindles = False
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
183
184
  for instance, iobj in config_data["instances"].items():
    for nic in iobj["nics"]:
185
186
187
188
189
190
191
192
      name = nic.get("network", None)
      if name:
        uuid = network2uuid.get(name, None)
        if uuid:
          print("NIC with network name %s found."
                " Substituting with uuid %s." % (name, uuid))
          nic["network"] = uuid

Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
193
194
195
196
197
198
199
200
201
202
203
    if "disks" not in iobj:
      raise Error("Instance '%s' doesn't have a disks entry?!" % instance)
    disks = iobj["disks"]
    for idx, dobj in enumerate(disks):
      expected = "disk/%s" % idx
      current = dobj.get("iv_name", "")
      if current != expected:
        logging.warning("Updating iv_name for instance %s/disk %s"
                        " from '%s' to '%s'",
                        instance, idx, current, expected)
        dobj["iv_name"] = expected
204
205
206
207
208
209
210
211
212
213
214
      if not "spindles" in dobj:
        missing_spindles = True

  if GetExclusiveStorageValue(config_data) and missing_spindles:
    # We cannot be sure that the instances that are missing spindles have
    # exclusive storage enabled (the check would be more complicated), so we
    # give a noncommittal message
    logging.warning("Some instance disks could be needing to update the"
                    " spindles parameter; you can check by running"
                    " 'gnt-cluster verify', and fix any problem with"
                    " 'gnt-cluster repair-disk-sizes'")
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287


def UpgradeRapiUsers():
  if (os.path.isfile(options.RAPI_USERS_FILE_PRE24) and
      not os.path.islink(options.RAPI_USERS_FILE_PRE24)):
    if os.path.exists(options.RAPI_USERS_FILE):
      raise Error("Found pre-2.4 RAPI users file at %s, but another file"
                  " already exists at %s" %
                  (options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE))
    logging.info("Found pre-2.4 RAPI users file at %s, renaming to %s",
                 options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE)
    if not options.dry_run:
      utils.RenameFile(options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE,
                       mkdir=True, mkdir_mode=0750)

  # Create a symlink for RAPI users file
  if (not (os.path.islink(options.RAPI_USERS_FILE_PRE24) or
           os.path.isfile(options.RAPI_USERS_FILE_PRE24)) and
      os.path.isfile(options.RAPI_USERS_FILE)):
    logging.info("Creating symlink from %s to %s",
                 options.RAPI_USERS_FILE_PRE24, options.RAPI_USERS_FILE)
    if not options.dry_run:
      os.symlink(options.RAPI_USERS_FILE, options.RAPI_USERS_FILE_PRE24)


def UpgradeWatcher():
  # Remove old watcher state file if it exists
  if os.path.exists(options.WATCHER_STATEFILE):
    logging.info("Removing watcher state file %s", options.WATCHER_STATEFILE)
    if not options.dry_run:
      utils.RemoveFile(options.WATCHER_STATEFILE)


def UpgradeFileStoragePaths(config_data):
  # Write file storage paths
  if not os.path.exists(options.FILE_STORAGE_PATHS_FILE):
    cluster = config_data["cluster"]
    file_storage_dir = cluster.get("file_storage_dir")
    shared_file_storage_dir = cluster.get("shared_file_storage_dir")
    del cluster

    logging.info("Ganeti 2.7 and later only allow whitelisted directories"
                 " for file storage; writing existing configuration values"
                 " into '%s'",
                 options.FILE_STORAGE_PATHS_FILE)

    if file_storage_dir:
      logging.info("File storage directory: %s", file_storage_dir)
    if shared_file_storage_dir:
      logging.info("Shared file storage directory: %s",
                   shared_file_storage_dir)

    buf = StringIO()
    buf.write("# List automatically generated from configuration by\n")
    buf.write("# cfgupgrade at %s\n" % time.asctime())
    if file_storage_dir:
      buf.write("%s\n" % file_storage_dir)
    if shared_file_storage_dir:
      buf.write("%s\n" % shared_file_storage_dir)
    utils.WriteFile(file_name=options.FILE_STORAGE_PATHS_FILE,
                    data=buf.getvalue(),
                    mode=0600,
                    dry_run=options.dry_run,
                    backup=True)


def UpgradeAll(config_data):
  config_data["version"] = constants.BuildVersion(TARGET_MAJOR,
                                                  TARGET_MINOR, 0)
  UpgradeRapiUsers()
  UpgradeWatcher()
  UpgradeFileStoragePaths(config_data)
  UpgradeNetworks(config_data)
288
  UpgradeCluster(config_data)
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
289
290
291
  UpgradeGroups(config_data)
  UpgradeInstances(config_data)

292

293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
def DowngradeDisks(disks, owner):
  for disk in disks:
    # Remove spindles to downgrade to 2.8
    if "spindles" in disk:
      logging.warning("Removing spindles (value=%s) from disk %s (%s) of"
                      " instance %s",
                      disk["spindles"], disk["iv_name"], disk["uuid"], owner)
      del disk["spindles"]


def DowngradeInstances(config_data):
  if "instances" not in config_data:
    raise Error("Cannot find the 'instances' key in the configuration!")
  for (iname, iobj) in config_data["instances"].items():
    if "disks" not in iobj:
      raise Error("Cannot find 'disks' key for instance %s" % iname)
    DowngradeDisks(iobj["disks"], iname)


312
313
314
def DowngradeAll(config_data):
  # Any code specific to a particular version should be labeled that way, so
  # it can be removed when updating to the next version.
315
  DowngradeInstances(config_data)
316
317


318
319
320
321
def main():
  """Main program.

  """
322
  global options, args # pylint: disable=W0603
323

324
  # Option parsing
325
  parser = optparse.OptionParser(usage="%prog [--debug|--verbose] [--force]")
326
  parser.add_option("--dry-run", dest="dry_run",
327
                    action="store_true",
328
329
                    help="Try to do the conversion, but don't write"
                         " output file")
330
  parser.add_option(cli.FORCE_OPT)
331
  parser.add_option(cli.DEBUG_OPT)
332
  parser.add_option(cli.VERBOSE_OPT)
333
334
335
  parser.add_option("--ignore-hostname", dest="ignore_hostname",
                    action="store_true", default=False,
                    help="Don't abort if hostname doesn't match")
336
  parser.add_option("--path", help="Convert configuration in this"
337
338
                    " directory instead of '%s'" % pathutils.DATA_DIR,
                    default=pathutils.DATA_DIR, dest="data_dir")
339
340
341
342
  parser.add_option("--confdir",
                    help=("Use this directory instead of '%s'" %
                          pathutils.CONF_DIR),
                    default=pathutils.CONF_DIR, dest="conf_dir")
343
344
345
  parser.add_option("--no-verify",
                    help="Do not verify configuration after upgrade",
                    action="store_true", dest="no_verify", default=False)
346
347
348
  parser.add_option("--downgrade",
                    help="Downgrade to the previous stable version",
                    action="store_true", dest="downgrade", default=False)
349
350
  (options, args) = parser.parse_args()

351
352
  # We need to keep filenames locally because they might be renamed between
  # versions.
353
  options.data_dir = os.path.abspath(options.data_dir)
354
355
356
357
  options.CONFIG_DATA_PATH = options.data_dir + "/config.data"
  options.SERVER_PEM_PATH = options.data_dir + "/server.pem"
  options.KNOWN_HOSTS_PATH = options.data_dir + "/known_hosts"
  options.RAPI_CERT_FILE = options.data_dir + "/rapi.pem"
358
359
  options.SPICE_CERT_FILE = options.data_dir + "/spice.pem"
  options.SPICE_CACERT_FILE = options.data_dir + "/spice-ca.pem"
360
361
  options.RAPI_USERS_FILE = options.data_dir + "/rapi/users"
  options.RAPI_USERS_FILE_PRE24 = options.data_dir + "/rapi_users"
362
  options.CONFD_HMAC_KEY = options.data_dir + "/hmac.key"
363
  options.CDS_FILE = options.data_dir + "/cluster-domain-secret"
364
  options.SSCONF_MASTER_NODE = options.data_dir + "/ssconf_master_node"
365
  options.WATCHER_STATEFILE = options.data_dir + "/watcher.data"
366
  options.FILE_STORAGE_PATHS_FILE = options.conf_dir + "/file-storage-paths"
367

368
369
  SetupLogging()

370
371
  # Option checking
  if args:
372
    raise Error("No arguments expected")
373
374
  if options.downgrade and not options.no_verify:
    options.no_verify = True
375

376
377
378
379
380
  # Check master name
  if not (CheckHostname(options.SSCONF_MASTER_NODE) or options.ignore_hostname):
    logging.error("Aborting due to hostname mismatch")
    sys.exit(constants.EXIT_FAILURE)

381
  if not options.force:
382
383
384
385
386
387
388
389
390
391
392
393
394
    if options.downgrade:
      usertext = ("The configuration is going to be DOWNGRADED to version %s.%s"
                  " Some configuration data might be removed if they don't fit"
                  " in the old format. Please make sure you have read the"
                  " upgrade notes (available in the UPGRADE file and included"
                  " in other documentation formats) to understand what they"
                  " are. Continue with *DOWNGRADING* the configuration?" %
                  (DOWNGRADE_MAJOR, DOWNGRADE_MINOR))
    else:
      usertext = ("Please make sure you have read the upgrade notes for"
                  " Ganeti %s (available in the UPGRADE file and included"
                  " in other documentation formats). Continue with upgrading"
                  " configuration?" % constants.RELEASE_VERSION)
395
    if not cli.AskUser(usertext):
396
      sys.exit(constants.EXIT_FAILURE)
397

398
  # Check whether it's a Ganeti configuration directory
399
  if not (os.path.isfile(options.CONFIG_DATA_PATH) and
400
          os.path.isfile(options.SERVER_PEM_PATH) and
401
          os.path.isfile(options.KNOWN_HOSTS_PATH)):
402
    raise Error(("%s does not seem to be a Ganeti configuration"
403
                 " directory") % options.data_dir)
404

405
406
407
  if not os.path.isdir(options.conf_dir):
    raise Error("Not a directory: %s" % options.conf_dir)

408
  config_data = serializer.LoadJson(utils.ReadFile(options.CONFIG_DATA_PATH))
409

410
411
412
413
  try:
    config_version = config_data["version"]
  except KeyError:
    raise Error("Unable to determine configuration version")
414

415
416
  (config_major, config_minor, config_revision) = \
    constants.SplitVersion(config_version)
417

418
419
  logging.info("Found configuration version %s (%d.%d.%d)",
               config_version, config_major, config_minor, config_revision)
420

421
422
423
  if "config_version" in config_data["cluster"]:
    raise Error("Inconsistent configuration: found config_version in"
                " configuration file")
424

425
426
427
428
429
430
431
432
433
  # Downgrade to the previous stable version
  if options.downgrade:
    if config_major != TARGET_MAJOR or config_minor != TARGET_MINOR:
      raise Error("Downgrade supported only from the latest version (%s.%s),"
                  " found %s (%s.%s.%s) instead" %
                  (TARGET_MAJOR, TARGET_MINOR, config_version, config_major,
                   config_minor, config_revision))
    DowngradeAll(config_data)

434
435
  # Upgrade from 2.{0..7} to 2.7
  elif config_major == 2 and config_minor in range(0, 8):
436
    if config_revision != 0:
437
      logging.warning("Config revision is %s, not 0", config_revision)
Bernardo Dal Seno's avatar
Bernardo Dal Seno committed
438
    UpgradeAll(config_data)
439

440
  elif config_major == TARGET_MAJOR and config_minor == TARGET_MINOR:
441
442
443
444
445
    logging.info("No changes necessary")

  else:
    raise Error("Configuration version %d.%d.%d not supported by this tool" %
                (config_major, config_minor, config_revision))
446

447
448
449
450
451
452
453
  try:
    logging.info("Writing configuration file to %s", options.CONFIG_DATA_PATH)
    utils.WriteFile(file_name=options.CONFIG_DATA_PATH,
                    data=serializer.DumpJson(config_data),
                    mode=0600,
                    dry_run=options.dry_run,
                    backup=True)
454
455

    if not options.dry_run:
Iustin Pop's avatar
Iustin Pop committed
456
457
458
459
460
461
462
463
      bootstrap.GenerateClusterCrypto(
        False, False, False, False, False,
        nodecert_file=options.SERVER_PEM_PATH,
        rapicert_file=options.RAPI_CERT_FILE,
        spicecert_file=options.SPICE_CERT_FILE,
        spicecacert_file=options.SPICE_CACERT_FILE,
        hmackey_file=options.CONFD_HMAC_KEY,
        cds_file=options.CDS_FILE)
464

465
  except Exception:
466
    logging.critical("Writing configuration failed. It is probably in an"
467
468
                     " inconsistent state and needs manual intervention.")
    raise
469

470
  # test loading the config file
471
  all_ok = True
472
  if not (options.dry_run or options.no_verify):
473
474
    logging.info("Testing the new config file...")
    cfg = config.ConfigWriter(cfg_file=options.CONFIG_DATA_PATH,
475
                              accept_foreign=options.ignore_hostname,
476
477
478
479
480
481
                              offline=True)
    # if we reached this, it's all fine
    vrfy = cfg.VerifyConfig()
    if vrfy:
      logging.error("Errors after conversion:")
      for item in vrfy:
Iustin Pop's avatar
Iustin Pop committed
482
        logging.error(" - %s", item)
483
484
485
      all_ok = False
    else:
      logging.info("File loaded successfully after upgrading")
486
487
    del cfg

488
489
490
491
492
493
  if options.downgrade:
    action = "downgraded"
    out_ver = "%s.%s" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR)
  else:
    action = "upgraded"
    out_ver = constants.RELEASE_VERSION
494
  if all_ok:
495
496
    cli.ToStderr("Configuration successfully %s to version %s.",
                 action, out_ver)
497
  else:
498
499
    cli.ToStderr("Configuration %s to version %s, but there are errors."
                 "\nPlease review the file.", action, out_ver)
500

501
502
503

if __name__ == "__main__":
  main()