Commit 723ec678 authored by Guido Trotter's avatar Guido Trotter
Browse files

Merge branch 'devel-2.7'



* devel-2.7: (26 commits)
  Fix burnin install path
  Fix format of the NEWS file
  NEWS: Add news entry for the hail disk policy fix
  Add shelltests verifying hail applies disk ipolicy per disk
  Make the disks parameter available to the constructor
  Verify individual disks in Instance
  Compatibility fix for GHC 7
  Properly update iv_name of disks while changing templates
  Limit the size of networks to /16
  Check minimum size of networks on creation
  Fix job queue directory permission problems
  The disk size of a diskless instance is 0, not None
  Postpone non-urgent TODO from 2.7 to 2.9
  Add tool for creating users and groups
  Backwards compatibility fix for Lucid
  Properly export errors while reading job list
  Fix typo and improve comment
  Fix a bug in the Runtime tests
  Restrict instance move to templates based on local files
  Introduce a constant for the copyable disk templates
  ...

Conflicts:
	NEWS: trivial
	lib/network.py: trivial
Signed-off-by: default avatarGuido Trotter <ultrotter@google.com>
Reviewed-by: default avatarHelga Velroyen <helgav@google.com>
parents 5914192c 7ee2098e
......@@ -102,6 +102,7 @@
/tools/kvm-ifup
/tools/burnin
/tools/ensure-dirs
/tools/users-setup
/tools/vcluster-setup
/tools/node-cleanup
/tools/node-daemon-setup
......
......@@ -190,6 +190,7 @@ CLEANFILES = \
$(man_MANS) \
$(manhtml) \
tools/kvm-ifup \
tools/users-setup \
tools/vcluster-setup \
stamp-directories \
stamp-srclinks \
......@@ -856,6 +857,7 @@ python_scripts = \
dist_tools_SCRIPTS = \
$(python_scripts) \
tools/burnin \
tools/kvm-console-wrapper \
tools/master-ip-setup \
tools/xen-console-wrapper
......@@ -865,6 +867,7 @@ nodist_tools_python_scripts = \
nodist_tools_SCRIPTS = \
$(nodist_tools_python_scripts) \
tools/users-setup \
tools/vcluster-setup
pkglib_python_scripts = \
......@@ -872,7 +875,6 @@ pkglib_python_scripts = \
tools/check-cert-expired
nodist_pkglib_python_scripts = \
tools/burnin \
tools/ensure-dirs \
tools/node-daemon-setup \
tools/prepare-node-join
......@@ -915,6 +917,7 @@ EXTRA_DIST = \
devel/upload \
devel/webserver \
tools/kvm-ifup.in \
tools/users-setup.in \
tools/vcluster-setup.in \
$(docinput) \
doc/html \
......@@ -990,6 +993,8 @@ TEST_FILES = \
test/data/htools/common-suffix.data \
test/data/htools/empty-cluster.data \
test/data/htools/hail-alloc-drbd.json \
test/data/htools/hail-alloc-invalid-twodisks.json \
test/data/htools/hail-alloc-twodisks.json \
test/data/htools/hail-change-group.json \
test/data/htools/hail-invalid-reloc.json \
test/data/htools/hail-node-evac.json \
......@@ -1294,6 +1299,10 @@ tools/kvm-ifup: tools/kvm-ifup.in $(REPLACE_VARS_SED)
sed -f $(REPLACE_VARS_SED) < $< > $@
chmod +x $@
tools/users-setup: tools/users-setup.in $(REPLACE_VARS_SED)
sed -f $(REPLACE_VARS_SED) < $< > $@
chmod +x $@
tools/vcluster-setup: tools/vcluster-setup.in $(REPLACE_VARS_SED)
sed -f $(REPLACE_VARS_SED) < $< > $@
chmod +x $@
......
......@@ -24,6 +24,53 @@ Version 2.8.0 beta1
configuration back to the previous stable version.
Version 2.7.0 rc1
-----------------
*(unreleased)*
- Fix hail to verify disk instance policies on a per-disk basis (Issue 418).
Version 2.7.0 beta2
-------------------
*(Released Tue, 2 Apr 2013)*
- Networks no longer have a "type" slot, since this information was
unused in Ganeti: instead of it tags should be used.
- Diskless instances are now externally mirrored (Issue 237). This for
now has only been tested in conjunction with explicit target nodes for
migration/failover.
- The rapi client now has a ``target_node`` option to MigrateInstance.
- Fix early exit return code for hbal (Issue 386).
- Fix ``gnt-instance migrate/failover -n`` (Issue 396).
- Fix ``rbd showmapped`` output parsing (Issue 312).
- Networks are now referenced indexed by UUID, rather than name. This
will require running cfgupgrade, from 2.7.0beta1, if networks are in
use.
- The OS environment now includes network information.
- Deleting of a network is now disallowed if any instance nic is using
it, to prevent dangling references.
- External storage is now documented in man pages.
- The exclusive_storage flag can now only be set at nodegroup level.
- Hbal can now submit an explicit priority with its jobs.
- Many network related locking fixes.
- Bump up the required pylint version to 0.25.1.
- Fix the ``no_remember`` option in RAPI client.
- Many ipolicy related tests, qa, and fixes.
- Many documentation improvements and fixes.
- Fix building with ``--disable-file-storage``.
- Fix ``-q`` option in htools, which was broken if passed more than
once.
- Some haskell/python interaction improvements and fixes.
- Fix iallocator in case of missing LVM storage.
- Fix confd config load in case of ``--no-lvm-storage``.
- The confd/query functionality is now mentioned in the security
documentation.
>>>>>>> devel-2.7
Version 2.7.0 beta1
-------------------
......
......@@ -51,6 +51,10 @@ To run commands on all nodes, the `distributed shell (dsh)
(``cfgupgrade`` supports a number of parameters, run it with
``--help`` for more information)
#. Upgrade the directory permissions on all nodes::
$ /usr/lib/ganeti/ensure-dirs --full-run
#. Restart daemons on all nodes::
$ /etc/init.d/ganeti restart
......
......@@ -113,6 +113,8 @@ def main():
m = RELEASED_RE.match(line)
if not m:
Error("Line %s: Invalid release line" % fileinput.filelineno())
expect_date = False
continue
# Including the weekday in the date string does not work as time.strptime
# would return an inconsistent result if the weekday is incorrect.
......
......@@ -2,7 +2,7 @@
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [7])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~beta1])
m4_define([gnt_version_suffix], [~beta2])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......
......@@ -3020,7 +3020,7 @@ def JobQueueUpdate(file_name, content):
# Write and replace the file atomically
utils.WriteFile(file_name, data=_Decompress(content), uid=getents.masterd_uid,
gid=getents.masterd_gid)
gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS)
def JobQueueRename(old, new):
......@@ -3044,8 +3044,8 @@ def JobQueueRename(old, new):
getents = runtime.GetEnts()
utils.RenameFile(old, new, mkdir=True, mkdir_mode=0700,
dir_uid=getents.masterd_uid, dir_gid=getents.masterd_gid)
utils.RenameFile(old, new, mkdir=True, mkdir_mode=0750,
dir_uid=getents.masterd_uid, dir_gid=getents.daemons_gid)
def BlockdevClose(instance_name, disks):
......
......@@ -1529,7 +1529,7 @@ def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status,
"INSTANCE_STATUS": status,
"INSTANCE_MINMEM": minmem,
"INSTANCE_MAXMEM": maxmem,
# TODO(2.7) remove deprecated "memory" value
# TODO(2.9) remove deprecated "memory" value
"INSTANCE_MEMORY": maxmem,
"INSTANCE_VCPUS": vcpus,
"INSTANCE_DISK_TEMPLATE": disk_template,
......@@ -8428,6 +8428,10 @@ class LUInstanceMove(LogicalUnit):
assert self.instance is not None, \
"Cannot retrieve locked instance %s" % self.op.instance_name
 
if instance.disk_template not in constants.DTS_COPYABLE:
raise errors.OpPrereqError("Disk template %s not suitable for copying" %
instance.disk_template, errors.ECODE_STATE)
node = self.cfg.GetNodeInfo(self.op.target_node)
assert node is not None, \
"Cannot retrieve locked node %s" % self.op.target_node
......@@ -8503,12 +8507,9 @@ class LUInstanceMove(LogicalUnit):
try:
_CreateDisks(self, instance, target_node=target_node)
except errors.OpExecError:
self.LogWarning("Device creation failed, reverting...")
try:
_RemoveDisks(self, instance, target_node=target_node)
finally:
self.cfg.ReleaseDRBDMinors(instance.name)
raise
self.LogWarning("Device creation failed")
self.cfg.ReleaseDRBDMinors(instance.name)
raise
 
cluster_name = self.cfg.GetClusterInfo().cluster_name
 
......@@ -9741,6 +9742,7 @@ def _CreateDisks(lu, instance, to_skip=None, target_node=None):
result.Raise("Failed to create directory '%s' on"
" node %s" % (file_storage_dir, pnode))
 
disks_created = []
# Note: this needs to be kept in sync with adding of disks in
# LUInstanceSetParams
for idx, device in enumerate(instance.disks):
......@@ -9750,7 +9752,19 @@ def _CreateDisks(lu, instance, to_skip=None, target_node=None):
#HARDCODE
for node in all_nodes:
f_create = node == pnode
_CreateBlockDev(lu, node, instance, device, f_create, info, f_create)
try:
_CreateBlockDev(lu, node, instance, device, f_create, info, f_create)
disks_created.append((node, device))
except errors.OpExecError:
logging.warning("Creating disk %s for instance '%s' failed",
idx, instance.name)
for (node, disk) in disks_created:
lu.cfg.SetDiskID(disk, node)
result = lu.rpc.call_blockdev_remove(node, disk)
if result.fail_msg:
logging.warning("Failed to remove newly-created disk %s on node %s:"
" %s", device, node, result.fail_msg)
raise
 
 
def _RemoveDisks(lu, instance, target_node=None, ignore_failures=False):
......@@ -9758,8 +9772,7 @@ def _RemoveDisks(lu, instance, target_node=None, ignore_failures=False):
 
This abstracts away some work from `AddInstance()` and
`RemoveInstance()`. Note that in case some of the devices couldn't
be removed, the removal will continue with the other ones (compare
with `_CreateDisks()`).
be removed, the removal will continue with the other ones.
 
@type lu: L{LogicalUnit}
@param lu: the logical unit on whose behalf we execute
......@@ -11033,12 +11046,9 @@ class LUInstanceCreate(LogicalUnit):
try:
_CreateDisks(self, iobj)
except errors.OpExecError:
self.LogWarning("Device creation failed, reverting...")
try:
_RemoveDisks(self, iobj)
finally:
self.cfg.ReleaseDRBDMinors(instance)
raise
self.LogWarning("Device creation failed")
self.cfg.ReleaseDRBDMinors(instance)
raise
 
feedback_fn("adding instance %s to cluster config" % instance)
 
......@@ -14100,6 +14110,7 @@ class LUInstanceSetParams(LogicalUnit):
# update instance structure
instance.disks = new_disks
instance.disk_template = constants.DT_PLAIN
_UpdateIvNames(0, instance.disks)
self.cfg.Update(instance, feedback_fn)
 
# Release locks in case removing disks takes a while
......
......@@ -533,6 +533,14 @@ DTS_FILEBASED = compat.UniqueFrozenset([
DT_SHARED_FILE,
])
# the set of disk templates that can be moved by copying
# Note: a requirement is that they're not accessed externally or shared between
# nodes; in particular, sharedfile is not suitable.
DTS_COPYABLE = compat.UniqueFrozenset([
DT_FILE,
DT_PLAIN,
])
# the set of disk templates that are supported by exclusive_storage
DTS_EXCL_STORAGE = compat.UniqueFrozenset([DT_PLAIN])
......@@ -1761,6 +1769,7 @@ NODE_EVAC_MODES = compat.UniqueFrozenset([
# Job queue
JOB_QUEUE_VERSION = 1
JOB_QUEUE_SIZE_HARD_LIMIT = 5000
JOB_QUEUE_FILES_PERMS = 0640
JOB_ID_TEMPLATE = r"\d+"
JOB_FILE_RE = re.compile(r"^job-(%s)$" % JOB_ID_TEMPLATE)
......
......@@ -1885,7 +1885,8 @@ class JobQueue(object):
"""
getents = runtime.GetEnts()
utils.WriteFile(file_name, data=data, uid=getents.masterd_uid,
gid=getents.masterd_gid)
gid=getents.daemons_gid,
mode=constants.JOB_QUEUE_FILES_PERMS)
if replicate:
names, addrs = self._GetNodeIp()
......
......@@ -111,7 +111,8 @@ def InitAndVerifyQueue(must_lock):
if version is None:
# Write new version file
utils.WriteFile(pathutils.JOB_QUEUE_VERSION_FILE,
uid=getents.masterd_uid, gid=getents.masterd_gid,
uid=getents.masterd_uid, gid=getents.daemons_gid,
mode=constants.JOB_QUEUE_FILES_PERMS,
data="%s\n" % constants.JOB_QUEUE_VERSION)
# Read again
......@@ -125,7 +126,8 @@ def InitAndVerifyQueue(must_lock):
if serial is None:
# Write new serial file
utils.WriteFile(pathutils.JOB_QUEUE_SERIAL_FILE,
uid=getents.masterd_uid, gid=getents.masterd_gid,
uid=getents.masterd_uid, gid=getents.daemons_gid,
mode=constants.JOB_QUEUE_FILES_PERMS,
data="%s\n" % 0)
# Read again
......@@ -174,7 +176,8 @@ def SetDrainFlag(drain_flag):
if drain_flag:
utils.WriteFile(pathutils.JOB_QUEUE_DRAIN_FILE, data="",
uid=getents.masterd_uid, gid=getents.masterd_gid)
uid=getents.masterd_uid, gid=getents.daemons_gid,
mode=constants.JOB_QUEUE_FILES_PERMS)
else:
utils.RemoveFile(pathutils.JOB_QUEUE_DRAIN_FILE)
......
......@@ -1621,7 +1621,7 @@ def ComputeDiskSize(disk_template, disks):
"""
# Required free disk space as a function of disk and swap space
req_size_dict = {
constants.DT_DISKLESS: None,
constants.DT_DISKLESS: 0,
constants.DT_PLAIN: sum(d[constants.IDISK_SIZE] for d in disks),
# 128 MB are added for drbd metadata for each disk
constants.DT_DRBD8:
......
......@@ -29,8 +29,20 @@ from bitarray import bitarray
from ganeti import errors
def _ComputeIpv4NumHosts(network_size):
"""Derives the number of hosts in an IPv4 network from the size.
"""
return 2 ** (32 - network_size)
IPV4_NETWORK_MIN_SIZE = 30
IPV4_NETWORK_MIN_NUM_HOSTS = 2 ** (32 - IPV4_NETWORK_MIN_SIZE)
# FIXME: This limit is for performance reasons. Remove when refactoring
# for performance tuning was successful.
IPV4_NETWORK_MAX_SIZE = 16
IPV4_NETWORK_MIN_NUM_HOSTS = _ComputeIpv4NumHosts(IPV4_NETWORK_MIN_SIZE)
IPV4_NETWORK_MAX_NUM_HOSTS = _ComputeIpv4NumHosts(IPV4_NETWORK_MAX_SIZE)
class AddressPool(object):
......@@ -58,6 +70,13 @@ class AddressPool(object):
self.net = network
self.network = ipaddr.IPNetwork(self.net.network)
if self.network.numhosts > IPV4_NETWORK_MAX_NUM_HOSTS:
raise errors.AddressPoolError("A big network with %s host(s) is currently"
" not supported. please specify at most a"
" /%s network" %
(str(self.network.numhosts),
IPV4_NETWORK_MAX_SIZE))
if self.network.numhosts < IPV4_NETWORK_MIN_NUM_HOSTS:
raise errors.AddressPoolError("A network with only %s host(s) is too"
" small, please specify at least a /%s"
......
......@@ -1652,7 +1652,7 @@ class Cluster(TaggableObject):
wrongkeys = frozenset(self.ipolicy.keys()) - constants.IPOLICY_ALL_KEYS
if wrongkeys:
# These keys would be silently removed by FillIPolicy()
msg = ("Cluster instance policy contains spourious keys: %s" %
msg = ("Cluster instance policy contains spurious keys: %s" %
utils.CommaJoin(wrongkeys))
raise errors.ConfigurationError(msg)
self.ipolicy = FillIPolicy(constants.IPOLICY_DEFAULTS, self.ipolicy)
......
......@@ -159,19 +159,19 @@ def GetPaths():
getent.noded_uid, getent.noded_gid, False))
paths.extend([
(pathutils.QUEUE_DIR, DIR, 0700, getent.masterd_uid, getent.masterd_gid),
(pathutils.QUEUE_DIR, QUEUE_DIR, 0600,
getent.masterd_uid, getent.masterd_gid),
(pathutils.QUEUE_DIR, DIR, 0750, getent.masterd_uid, getent.daemons_gid),
(pathutils.QUEUE_DIR, QUEUE_DIR, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid),
(pathutils.JOB_QUEUE_DRAIN_FILE, FILE, 0644,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.JOB_QUEUE_LOCK_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.JOB_QUEUE_SERIAL_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.JOB_QUEUE_VERSION_FILE, FILE, 0600,
getent.masterd_uid, getent.masterd_gid, False),
(pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0700,
getent.masterd_uid, getent.masterd_gid),
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_LOCK_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_SERIAL_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_VERSION_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS,
getent.masterd_uid, getent.daemons_gid, False),
(pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0740,
getent.masterd_uid, getent.daemons_gid),
(rapi_dir, DIR, 0750, getent.rapi_uid, getent.masterd_gid),
(pathutils.RAPI_USERS_FILE, FILE, 0640,
getent.rapi_uid, getent.masterd_gid, False),
......@@ -246,7 +246,7 @@ def Main():
if opts.full_run:
RecursiveEnsure(pathutils.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid,
getent.masterd_gid, 0700, 0600)
getent.daemons_gid, 0750, constants.JOB_QUEUE_FILES_PERMS)
except errors.GenericError, err:
logging.error("An error occurred while setting permissions: %s", err)
return constants.EXIT_FAILURE
......
......@@ -65,14 +65,17 @@ parseBaseInstance :: String
-> JSRecord
-> Result (String, Instance.Instance)
parseBaseInstance n a = do
let extract x = tryFromObj ("invalid data for instance '" ++ n ++ "'") a x
let errorMessage = "invalid data for instance '" ++ n ++ "'"
let extract x = tryFromObj errorMessage a x
disk <- extract "disk_space_total"
disks <- extract "disks" >>= toArray >>= asObjectList >>=
mapM (flip (tryFromObj errorMessage) "size" . fromJSObject)
mem <- extract "memory"
vcpus <- extract "vcpus"
tags <- extract "tags"
dt <- extract "disk_template"
su <- extract "spindle_use"
return (n, Instance.create n mem disk vcpus Running tags True 0 0 dt su)
return (n, Instance.create n mem disk disks vcpus Running tags True 0 0 dt su)
-- | Parses an instance as found in the cluster instance list.
parseInstance :: NameAssoc -- ^ The node name-to-index association list
......
......@@ -172,7 +172,7 @@ parseInstance ktn [ name, disk, mem, vcpus
xauto_balance <- convert "auto_balance" auto_balance
xdt <- convert "disk_template" disk_template
xsu <- convert "be/spindle_use" su
let inst = Instance.create xname xmem xdisk xvcpus
let inst = Instance.create xname xmem xdisk [xdisk] xvcpus
xrunning xtags xauto_balance xpnode snode xdt xsu
return (xname, inst)
......
......@@ -122,6 +122,7 @@ parseInstance ktn a = do
let owner_name = "Instance '" ++ name ++ "', error while parsing data"
let extract s x = tryFromObj owner_name x s
disk <- extract "disk_usage" a
disks <- extract "disk.sizes" a
beparams <- liftM fromJSObject (extract "beparams" a)
omem <- extract "oper_ram" a
mem <- case omem of
......@@ -138,7 +139,7 @@ parseInstance ktn a = do
auto_balance <- extract "auto_balance" beparams
dt <- extract "disk_template" a
su <- extract "spindle_use" beparams
let inst = Instance.create name mem disk vcpus running tags
let inst = Instance.create name mem disk disks vcpus running tags
auto_balance pnode snode dt su
return (name, inst)
......
......@@ -231,7 +231,7 @@ loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode
when (sidx == pidx) . fail $ "Instance " ++ name ++
" has same primary and secondary node - " ++ pnode
let vtags = commaSplit tags
newinst = Instance.create name vmem vdsk vvcpus vstatus vtags
newinst = Instance.create name vmem vdsk [vdsk] vvcpus vstatus vtags
auto_balance pidx sidx disk_template spindle_use
return (name, newinst)
......
......@@ -70,7 +70,8 @@ data Instance = Instance
{ name :: String -- ^ The instance name
, alias :: String -- ^ The shortened name
, mem :: Int -- ^ Memory of the instance
, dsk :: Int -- ^ Disk size of instance
, dsk :: Int -- ^ Total disk usage of the instance
, disks :: [Int] -- ^ Sizes of the individual disks
, vcpus :: Int -- ^ Number of VCPUs
, runSt :: T.InstanceStatus -- ^ Original run status
, pNode :: T.Ndx -- ^ Original primary node
......@@ -162,15 +163,16 @@ type List = Container.Container Instance
--
-- Some parameters are not initialized by function, and must be set
-- later (via 'setIdx' for example).
create :: String -> Int -> Int -> Int -> T.InstanceStatus
create :: String -> Int -> Int -> [Int] -> Int -> T.InstanceStatus
-> [String] -> Bool -> T.Ndx -> T.Ndx -> T.DiskTemplate -> Int
-> Instance
create name_init mem_init dsk_init vcpus_init run_init tags_init
create name_init mem_init dsk_init disks_init vcpus_init run_init tags_init
auto_balance_init pn sn dt su =
Instance { name = name_init
, alias = name_init
, mem = mem_init
, dsk = dsk_init
, disks = disks_init
, vcpus = vcpus_init
, runSt = run_init
, pNode = pn
......@@ -265,7 +267,7 @@ specOf Instance { mem = m, dsk = d, vcpus = c } =
instBelowISpec :: Instance -> T.ISpec -> T.OpResult ()
instBelowISpec inst ispec
| mem inst > T.iSpecMemorySize ispec = Bad T.FailMem
| dsk inst > T.iSpecDiskSize ispec = Bad T.FailDisk
| any (> T.iSpecDiskSize ispec) (disks inst) = Bad T.FailDisk
| vcpus inst > T.iSpecCpuCount ispec = Bad T.FailCPU
| otherwise = Ok ()
......@@ -273,7 +275,7 @@ instBelowISpec inst ispec
instAboveISpec :: Instance -> T.ISpec -> T.OpResult ()
instAboveISpec inst ispec
| mem inst < T.iSpecMemorySize ispec = Bad T.FailMem
| dsk inst < T.iSpecDiskSize ispec = Bad T.FailDisk
| any (< T.iSpecDiskSize ispec) (disks inst) = Bad T.FailDisk
| vcpus inst < T.iSpecCpuCount ispec = Bad T.FailCPU
| otherwise = Ok ()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment