From 8106dd64465f76258a75fd618edbf5b944ce72be Mon Sep 17 00:00:00 2001 From: Santi Raffa Date: Tue, 17 Dec 2013 10:28:35 +0100 Subject: [PATCH] Gluster: minimal implementation Add Gluster to Ganeti by essentially cloning the shared file behaviour everywhere in the code base. Signed-off-by: Santi Raffa Signed-off-by: Thomas Thrainer Reviewed-by: Thomas Thrainer --- Makefile.am | 4 +- lib/cmdlib/instance.py | 5 +- lib/cmdlib/instance_storage.py | 1 + lib/masterd/instance.py | 1 + lib/objects.py | 5 +- lib/storage/bdev.py | 2 + lib/storage/gluster.py | 158 +++++++++++++++++++++ lib/tools/burnin.py | 3 + lib/utils/storage.py | 2 + src/Ganeti/Constants.hs | 29 +++- src/Ganeti/HTools/Cluster.hs | 6 + src/Ganeti/HTools/Instance.hs | 1 + src/Ganeti/HTools/Types.hs | 1 + src/Ganeti/Objects.hs | 7 + src/Ganeti/Storage/Utils.hs | 1 + src/Ganeti/Types.hs | 2 + test/py/ganeti.objects_unittest.py | 2 +- test/py/ganeti.storage.gluster_unittest.py | 26 ++++ 18 files changed, 243 insertions(+), 13 deletions(-) create mode 100644 lib/storage/gluster.py create mode 100644 test/py/ganeti.storage.gluster_unittest.py diff --git a/Makefile.am b/Makefile.am index b3414da5a..24d70bc40 100644 --- a/Makefile.am +++ b/Makefile.am @@ -421,7 +421,8 @@ storage_PYTHON = \ lib/storage/drbd.py \ lib/storage/drbd_info.py \ lib/storage/drbd_cmdgen.py \ - lib/storage/filestorage.py + lib/storage/filestorage.py \ + lib/storage/gluster.py rapi_PYTHON = \ lib/rapi/__init__.py \ @@ -1445,6 +1446,7 @@ python_tests = \ test/py/ganeti.storage.container_unittest.py \ test/py/ganeti.storage.drbd_unittest.py \ test/py/ganeti.storage.filestorage_unittest.py \ + test/py/ganeti.storage.gluster_unittest.py \ test/py/ganeti.tools.burnin_unittest.py \ test/py/ganeti.tools.ensure_dirs_unittest.py \ test/py/ganeti.tools.node_daemon_setup_unittest.py \ diff --git a/lib/cmdlib/instance.py b/lib/cmdlib/instance.py index 9117f8c1b..4f512a4e6 100644 --- a/lib/cmdlib/instance.py +++ b/lib/cmdlib/instance.py @@ -849,7 +849,8 @@ class LUInstanceCreate(LogicalUnit): # build the full file storage dir path joinargs = [] - if self.op.disk_template == constants.DT_SHARED_FILE: + if self.op.disk_template in (constants.DT_SHARED_FILE, + constants.DT_GLUSTER): get_fsd_fn = self.cfg.GetSharedFileStorageDir else: get_fsd_fn = self.cfg.GetFileStorageDir @@ -1754,7 +1755,7 @@ class LUInstanceMove(LogicalUnit): for idx, dsk in enumerate(self.instance.disks): if dsk.dev_type not in (constants.DT_PLAIN, constants.DT_FILE, - constants.DT_SHARED_FILE): + constants.DT_SHARED_FILE, constants.DT_GLUSTER): raise errors.OpPrereqError("Instance disk %d has a complex layout," " cannot copy" % idx, errors.ECODE_STATE) diff --git a/lib/cmdlib/instance_storage.py b/lib/cmdlib/instance_storage.py index eb022ce09..641b25383 100644 --- a/lib/cmdlib/instance_storage.py +++ b/lib/cmdlib/instance_storage.py @@ -285,6 +285,7 @@ def ComputeDiskSizePerVG(disk_template, disks): constants.DT_DRBD8: _compute(disks, constants.DRBD_META_SIZE), constants.DT_FILE: {}, constants.DT_SHARED_FILE: {}, + constants.DT_GLUSTER: {}, } if disk_template not in req_size_dict: diff --git a/lib/masterd/instance.py b/lib/masterd/instance.py index 8cac3a3af..1f90e7e4f 100644 --- a/lib/masterd/instance.py +++ b/lib/masterd/instance.py @@ -1648,6 +1648,7 @@ def ComputeDiskSize(disk_template, disks): sum(d[constants.IDISK_SIZE] + constants.DRBD_META_SIZE for d in disks), constants.DT_FILE: sum(d[constants.IDISK_SIZE] for d in disks), constants.DT_SHARED_FILE: sum(d[constants.IDISK_SIZE] for d in disks), + constants.DT_GLUSTER: sum(d[constants.IDISK_SIZE] for d in disks), constants.DT_BLOCK: 0, constants.DT_RBD: sum(d[constants.IDISK_SIZE] for d in disks), constants.DT_EXT: sum(d[constants.IDISK_SIZE] for d in disks), diff --git a/lib/objects.py b/lib/objects.py index ea88a9d6e..17d27bf03 100644 --- a/lib/objects.py +++ b/lib/objects.py @@ -586,7 +586,8 @@ class Disk(ConfigObject): """ if self.dev_type in [constants.DT_PLAIN, constants.DT_FILE, constants.DT_BLOCK, constants.DT_RBD, - constants.DT_EXT, constants.DT_SHARED_FILE]: + constants.DT_EXT, constants.DT_SHARED_FILE, + constants.DT_GLUSTER]: result = [node_uuid] elif self.dev_type in constants.DTS_DRBD: result = [self.logical_id[0], self.logical_id[1]] @@ -663,7 +664,7 @@ class Disk(ConfigObject): """ if self.dev_type in (constants.DT_PLAIN, constants.DT_FILE, constants.DT_RBD, constants.DT_EXT, - constants.DT_SHARED_FILE): + constants.DT_SHARED_FILE, constants.DT_GLUSTER): self.size += amount elif self.dev_type == constants.DT_DRBD8: if self.children: diff --git a/lib/storage/bdev.py b/lib/storage/bdev.py index 8d0c35b46..075b24f19 100644 --- a/lib/storage/bdev.py +++ b/lib/storage/bdev.py @@ -39,6 +39,7 @@ from ganeti import serializer from ganeti.storage import base from ganeti.storage import drbd from ganeti.storage.filestorage import FileStorage +from ganeti.storage.gluster import GlusterStorage class RbdShowmappedJsonError(Exception): @@ -1667,6 +1668,7 @@ DEV_MAP = { constants.DT_EXT: ExtStorageDevice, constants.DT_FILE: FileStorage, constants.DT_SHARED_FILE: FileStorage, + constants.DT_GLUSTER: GlusterStorage, } """Map disk types to disk type classes. diff --git a/lib/storage/gluster.py b/lib/storage/gluster.py new file mode 100644 index 000000000..e90fdacec --- /dev/null +++ b/lib/storage/gluster.py @@ -0,0 +1,158 @@ +# +# + +# Copyright (C) 2013 Google Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +"""Gluster storage class. + +This class is very similar to FileStorage, given that Gluster when mounted +behaves essentially like a regular file system. Unlike RBD, there are no +special provisions for block device abstractions (yet). + +""" +from ganeti import errors + +from ganeti.storage import base +from ganeti.storage.filestorage import FileDeviceHelper + + +class GlusterStorage(base.BlockDev): + """File device using the Gluster backend. + + This class represents a file storage backend device stored on Gluster. The + system administrator must mount the Gluster device himself at boot time before + Ganeti is run. + + The unique_id for the file device is a (file_driver, file_path) tuple. + + """ + def __init__(self, unique_id, children, size, params, dyn_params): + """Initalizes a file device backend. + + """ + if children: + base.ThrowError("Invalid setup for file device") + super(GlusterStorage, self).__init__(unique_id, children, size, params, + dyn_params) + if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: + raise ValueError("Invalid configuration data %s" % str(unique_id)) + self.driver = unique_id[0] + self.dev_path = unique_id[1] + + self.file = FileDeviceHelper(self.dev_path) + + self.Attach() + + def Assemble(self): + """Assemble the device. + + Checks whether the file device exists, raises BlockDeviceError otherwise. + + """ + assert self.attached, "Gluster file assembled without being attached" + self.file.Exists(assert_exists=True) + + def Shutdown(self): + """Shutdown the device. + + """ + + self.file = None + self.dev_path = None + self.attached = False + + def Open(self, force=False): + """Make the device ready for I/O. + + This is a no-op for the file type. + + """ + assert self.attached, "Gluster file opened without being attached" + + def Close(self): + """Notifies that the device will no longer be used for I/O. + + This is a no-op for the file type. + """ + pass + + def Remove(self): + """Remove the file backing the block device. + + @rtype: boolean + @return: True if the removal was successful + + """ + return self.file.Remove() + + def Rename(self, new_id): + """Renames the file. + + """ + # TODO: implement rename for file-based storage + base.ThrowError("Rename is not supported for Gluster storage") + + def Grow(self, amount, dryrun, backingstore, excl_stor): + """Grow the file + + @param amount: the amount (in mebibytes) to grow with + + """ + self.file.Grow(amount, dryrun, backingstore, excl_stor) + + def Attach(self): + """Attach to an existing file. + + Check if this file already exists. + + @rtype: boolean + @return: True if file exists + + """ + self.attached = self.file.Exists() + return self.attached + + def GetActualSize(self): + """Return the actual disk size. + + @note: the device needs to be active when this is called + + """ + return self.file.Size() + + @classmethod + def Create(cls, unique_id, children, size, spindles, params, excl_stor, + dyn_params): + """Create a new file. + + @param size: the size of file in MiB + + @rtype: L{bdev.FileStorage} + @return: an instance of FileStorage + + """ + if excl_stor: + raise errors.ProgrammerError("FileStorage device requested with" + " exclusive_storage") + if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: + raise ValueError("Invalid configuration data %s" % str(unique_id)) + + dev_path = unique_id[1] + + FileDeviceHelper.Create(dev_path, size) + return GlusterStorage(unique_id, children, size, params, dyn_params) diff --git a/lib/tools/burnin.py b/lib/tools/burnin.py index 190b9fe93..1863a1bd9 100755 --- a/lib/tools/burnin.py +++ b/lib/tools/burnin.py @@ -61,6 +61,7 @@ _SINGLE_NODE_DISK_TEMPLATES = compat.UniqueFrozenset([ constants.DT_SHARED_FILE, constants.DT_EXT, constants.DT_RBD, + constants.DT_GLUSTER ]) _SUPPORTED_DISK_TEMPLATES = compat.UniqueFrozenset([ @@ -71,6 +72,7 @@ _SUPPORTED_DISK_TEMPLATES = compat.UniqueFrozenset([ constants.DT_PLAIN, constants.DT_RBD, constants.DT_SHARED_FILE, + constants.DT_GLUSTER ]) #: Disk templates for which import/export is tested @@ -78,6 +80,7 @@ _IMPEXP_DISK_TEMPLATES = (_SUPPORTED_DISK_TEMPLATES - frozenset([ constants.DT_DISKLESS, constants.DT_FILE, constants.DT_SHARED_FILE, + constants.DT_GLUSTER ])) diff --git a/lib/utils/storage.py b/lib/utils/storage.py index 7cba91f1c..bf5695342 100644 --- a/lib/utils/storage.py +++ b/lib/utils/storage.py @@ -91,6 +91,8 @@ def _GetDefaultStorageUnitForDiskTemplate(cfg, disk_template): return (storage_type, cluster.file_storage_dir) elif disk_template == constants.DT_SHARED_FILE: return (storage_type, cluster.shared_file_storage_dir) + elif disk_template == constants.DT_GLUSTER: + return (storage_type, constants.GLUSTER_MOUNTPOINT) else: return (storage_type, None) diff --git a/src/Ganeti/Constants.hs b/src/Ganeti/Constants.hs index 6641be6fa..da2f64085 100644 --- a/src/Ganeti/Constants.hs +++ b/src/Ganeti/Constants.hs @@ -789,6 +789,9 @@ dtRbd = Types.diskTemplateToRaw DTRbd dtExt :: String dtExt = Types.diskTemplateToRaw DTExt +dtGluster :: String +dtGluster = Types.diskTemplateToRaw DTGluster + -- | This is used to order determine the default disk template when -- the list of enabled disk templates is inferred from the current -- state of the cluster. This only happens on an upgrade from a @@ -797,7 +800,8 @@ dtExt = Types.diskTemplateToRaw DTExt diskTemplatePreference :: [String] diskTemplatePreference = map Types.diskTemplateToRaw - [DTBlock, DTDiskless, DTDrbd8, DTExt, DTFile, DTPlain, DTRbd, DTSharedFile] + [DTBlock, DTDiskless, DTDrbd8, DTExt, DTFile, + DTPlain, DTRbd, DTSharedFile, DTGluster] diskTemplates :: FrozenSet String diskTemplates = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [minBound..] @@ -818,7 +822,8 @@ mapDiskTemplateStorageType = (DTFile, StorageFile), (DTDiskless, StorageDiskless), (DTPlain, StorageLvmVg), - (DTRbd, StorageRados)] + (DTRbd, StorageRados), + (DTGluster, StorageFile)] -- | The set of network-mirrored disk templates dtsIntMirror :: FrozenSet String @@ -828,21 +833,22 @@ dtsIntMirror = ConstantUtils.mkSet [dtDrbd8] dtsExtMirror :: FrozenSet String dtsExtMirror = ConstantUtils.mkSet $ - map Types.diskTemplateToRaw [DTDiskless, DTBlock, DTExt, DTSharedFile, DTRbd] + map Types.diskTemplateToRaw + [DTDiskless, DTBlock, DTExt, DTSharedFile, DTRbd, DTGluster] -- | The set of non-lvm-based disk templates dtsNotLvm :: FrozenSet String dtsNotLvm = ConstantUtils.mkSet $ map Types.diskTemplateToRaw - [DTSharedFile, DTDiskless, DTBlock, DTExt, DTFile, DTRbd] + [DTSharedFile, DTDiskless, DTBlock, DTExt, DTFile, DTRbd, DTGluster] -- | The set of disk templates which can be grown dtsGrowable :: FrozenSet String dtsGrowable = ConstantUtils.mkSet $ map Types.diskTemplateToRaw - [DTSharedFile, DTDrbd8, DTPlain, DTExt, DTFile, DTRbd] + [DTSharedFile, DTDrbd8, DTPlain, DTExt, DTFile, DTRbd, DTGluster] -- | The set of disk templates that allow adoption dtsMayAdopt :: FrozenSet String @@ -860,7 +866,8 @@ dtsMirrored = dtsIntMirror `ConstantUtils.union` dtsExtMirror -- | The set of file based disk templates dtsFilebased :: FrozenSet String dtsFilebased = - ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTSharedFile, DTFile] + ConstantUtils.mkSet $ map Types.diskTemplateToRaw + [DTSharedFile, DTFile, DTGluster] -- | The set of disk templates that can be moved by copying -- @@ -878,7 +885,7 @@ dtsExclStorage = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTPlain] dtsNoFreeSpaceCheck :: FrozenSet String dtsNoFreeSpaceCheck = ConstantUtils.mkSet $ - map Types.diskTemplateToRaw [DTExt, DTSharedFile, DTFile, DTRbd] + map Types.diskTemplateToRaw [DTExt, DTSharedFile, DTFile, DTRbd, DTGluster] dtsBlock :: FrozenSet String dtsBlock = @@ -3763,6 +3770,7 @@ diskLdDefaults = , (ldpAccess, PyValueEx diskKernelspace) ]) , (DTSharedFile, Map.empty) + , (DTGluster, Map.empty) ] diskDtDefaults :: Map DiskTemplate (Map String PyValueEx) @@ -3795,6 +3803,7 @@ diskDtDefaults = , (rbdAccess, PyValueEx diskKernelspace) ]) , (DTSharedFile, Map.empty) + , (DTGluster, Map.empty) ] niccDefaults :: Map String PyValueEx @@ -4577,3 +4586,9 @@ errorsEcodeAll = jstoreJobsPerArchiveDirectory :: Int jstoreJobsPerArchiveDirectory = 10000 + +-- * Gluster settings + +-- | Where Ganeti should manage Gluster volume mountpoints +glusterMountpoint :: String +glusterMountpoint = "/var/run/ganeti/gluster" diff --git a/src/Ganeti/HTools/Cluster.hs b/src/Ganeti/HTools/Cluster.hs index 054a8dc57..88067ab4c 100644 --- a/src/Ganeti/HTools/Cluster.hs +++ b/src/Ganeti/HTools/Cluster.hs @@ -982,6 +982,12 @@ nodeEvacInstance nl il mode inst@(Instance.Instance failOnSecondaryChange mode dt >> evacOneNodeOnly nl il inst gdx avail_nodes +nodeEvacInstance nl il mode inst@(Instance.Instance + {Instance.diskTemplate = dt@DTGluster}) + gdx avail_nodes = + failOnSecondaryChange mode dt >> + evacOneNodeOnly nl il inst gdx avail_nodes + nodeEvacInstance nl il ChangePrimary inst@(Instance.Instance {Instance.diskTemplate = DTDrbd8}) _ _ = diff --git a/src/Ganeti/HTools/Instance.hs b/src/Ganeti/HTools/Instance.hs index 5416425fc..92e414d6c 100644 --- a/src/Ganeti/HTools/Instance.hs +++ b/src/Ganeti/HTools/Instance.hs @@ -157,6 +157,7 @@ movableDiskTemplates = [ T.DTDrbd8 , T.DTBlock , T.DTSharedFile + , T.DTGluster , T.DTRbd , T.DTExt ] diff --git a/src/Ganeti/HTools/Types.hs b/src/Ganeti/HTools/Types.hs index 141164db4..fd4fabc5e 100644 --- a/src/Ganeti/HTools/Types.hs +++ b/src/Ganeti/HTools/Types.hs @@ -140,6 +140,7 @@ templateMirrorType DTBlock = MirrorExternal templateMirrorType DTDrbd8 = MirrorInternal templateMirrorType DTRbd = MirrorExternal templateMirrorType DTExt = MirrorExternal +templateMirrorType DTGluster = MirrorExternal -- | The resource spec type. data RSpec = RSpec diff --git a/src/Ganeti/Objects.hs b/src/Ganeti/Objects.hs index ce8241e85..78aa78d55 100644 --- a/src/Ganeti/Objects.hs +++ b/src/Ganeti/Objects.hs @@ -353,6 +353,13 @@ decodeDLId obj lid = do path' <- readJSON path return $ LIDSharedFile driver' path' _ -> fail "Can't read logical_id for shared file type" + DTGluster -> + case lid of + JSArray [driver, path] -> do + driver' <- readJSON driver + path' <- readJSON path + return $ LIDSharedFile driver' path' + _ -> fail "Can't read logical_id for shared file type" DTBlock -> case lid of JSArray [driver, path] -> do diff --git a/src/Ganeti/Storage/Utils.hs b/src/Ganeti/Storage/Utils.hs index 64fd19091..ef0df37c9 100644 --- a/src/Ganeti/Storage/Utils.hs +++ b/src/Ganeti/Storage/Utils.hs @@ -31,6 +31,7 @@ module Ganeti.Storage.Utils import Ganeti.Config import Ganeti.Objects import Ganeti.Types +import Ganeti.Constants import qualified Ganeti.Types as T import Control.Monad diff --git a/src/Ganeti/Types.hs b/src/Ganeti/Types.hs index 94b190a95..3200336dc 100644 --- a/src/Ganeti/Types.hs +++ b/src/Ganeti/Types.hs @@ -297,6 +297,7 @@ $(THH.declareLADT ''String "DiskTemplate" , ("DTDrbd8", "drbd") , ("DTRbd", "rbd") , ("DTExt", "ext") + , ("DTGluster", "gluster") ]) $(THH.makeJSONInstance ''DiskTemplate) @@ -530,6 +531,7 @@ diskTemplateToStorageType DTPlain = StorageLvmVg diskTemplateToStorageType DTRbd = StorageRados diskTemplateToStorageType DTDiskless = StorageDiskless diskTemplateToStorageType DTBlock = StorageBlock +diskTemplateToStorageType DTGluster = StorageFile -- | Equips a raw storage unit with its parameters addParamsToStorageUnit :: SPExclusiveStorage -> StorageUnitRaw -> StorageUnit diff --git a/test/py/ganeti.objects_unittest.py b/test/py/ganeti.objects_unittest.py index ff8a299d9..1fcee13a4 100755 --- a/test/py/ganeti.objects_unittest.py +++ b/test/py/ganeti.objects_unittest.py @@ -746,7 +746,7 @@ class TestDisk(unittest.TestCase): def testUpgradeConfigDevTypeLegacyUnchanged(self): dev_types = [constants.DT_FILE, constants.DT_SHARED_FILE, constants.DT_BLOCK, constants.DT_EXT, - constants.DT_RBD] + constants.DT_RBD, constants.DT_GLUSTER] for dev_type in dev_types: disk = objects.Disk() disk.dev_type = dev_type diff --git a/test/py/ganeti.storage.gluster_unittest.py b/test/py/ganeti.storage.gluster_unittest.py new file mode 100644 index 000000000..6814cb900 --- /dev/null +++ b/test/py/ganeti.storage.gluster_unittest.py @@ -0,0 +1,26 @@ +#!/usr/bin/python +# + +# Copyright (C) 2013 Google Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +"""Script for unittesting the ganeti.storage.gluster module""" + +import testutils + +if __name__ == "__main__": + testutils.GanetiTestProgram() -- GitLab