Commit 7803e37c authored by Iustin Pop's avatar Iustin Pop
Browse files

Merge branch 'stable-2.6'



* stable-2.6: (21 commits)
  Release Ganeti 2.6.0 rc4
  Prepare NEWS for Ganeti 2.6.0rc4
  Add some rudimentary node group ipolicy checks
  Fix setting ipolicy on node groups
  Fix --no-headers for the new list-drbd command
  Add a simple QA test for gnt-node list-drbd
  Add a new gnt-node command list-drbd
  Implement a node to drbd minors query function
  Add a new unused confd query
  Add support for computing instance all/secondary nodes
  Add disk logical ID support in Objects.hs
  Fix a docstring in bdev's DRBD8 class
  Extend the Template Haskell loadFn model
  Change how customFields are built
  Implement lookup of partial names in Config.hs
  Reorganise the lookup functions
  Remove an unused function
  Ensure that disk.params is always defined (and a dict)
  Another small consistency fix with if branches
  Fix inconsistency in the LUXI protocol w.r.t. args
  ...

Conflicts:
        htools/Ganeti/HTools/QC.hs (imports changes, trivial)
Signed-off-by: default avatarIustin Pop <iustin@google.com>
Reviewed-by: default avatarRené Nussbaumer <rn@google.com>
parents 1e00889c 95e7e676
......@@ -2,10 +2,10 @@ News
====
Version 2.6.0 rc3
Version 2.6.0 rc4
-----------------
*(Released Fri, 13 Jul 2012)*
*(Released Thu, 19 Jul 2012)*
New features
~~~~~~~~~~~~
......@@ -222,6 +222,9 @@ daemon will result in a smaller footprint; for larger systems, we
welcome feedback on the Haskell version which might become the default
in future versions.
If you want to use ``gnt-node list-drbd`` you need to have the Haskell
daemon running. The Python version doesn't implement the new call.
User interface changes
~~~~~~~~~~~~~~~~~~~~~~
......@@ -244,6 +247,10 @@ Also the handling of instances with regard to offline secondaries has
been improved. Instance operations should not fail because one of it's
secondary nodes is offline, even though it's safe to proceed.
A new command ``list-drbd`` has been added to the ``gnt-node`` script to
support debugging of DRBD issues on nodes. It provides a mapping of DRBD
minors to instance name.
API changes
~~~~~~~~~~~
......@@ -298,6 +305,28 @@ changed to allow the same clock skew as permitted by the cluster
verification. This will remove some rare but hard to diagnose errors in
import-export.
The ``LUXI`` protocol has been made more consistent regarding its
handling of command arguments. This, however, leads to incompatibility
issues with previous versions. Please ensure that you restart Ganeti
daemons after the upgrade, otherwise job submission will fail.
Version 2.6.0 rc3
-----------------
*(Released Fri, 13 Jul 2012)*
Third release candidate for 2.6. The following changes were done from
rc3 to rc4:
- Fixed ``UpgradeConfig`` w.r.t. to disk parameters on disk objects.
- Fixed an inconsistency in the LUXI protocol with the provided
arguments (NOT backwards compatible)
- Fixed a bug with node groups ipolicy where ``min`` was greater than
the cluster ``std`` value
- Implemented a new ``gnt-node list-drbd`` call to list DRBD minors for
easier instance debugging on nodes (requires ``hconfd`` to work)
Version 2.6.0 rc2
-----------------
......
......@@ -2,7 +2,7 @@
m4_define([gnt_version_major], [2])
m4_define([gnt_version_minor], [6])
m4_define([gnt_version_revision], [0])
m4_define([gnt_version_suffix], [~rc3])
m4_define([gnt_version_suffix], [~rc4])
m4_define([gnt_version_full],
m4_format([%d.%d.%d%s],
gnt_version_major, gnt_version_minor,
......
......@@ -26,9 +26,19 @@ module Ganeti.BasicTypes
, eitherToResult
, annotateResult
, annotateIOError
, select
, LookupResult(..)
, MatchPriority(..)
, lookupName
, goodLookupResult
, goodMatchPriority
, prefixMatch
, compareNameComponent
) where
import Control.Monad
import Data.Function
import Data.List
-- | This is similar to the JSON library Result type - /very/ similar,
-- but we want to use it in multiple places, so we abstract it into a
......@@ -78,3 +88,85 @@ annotateResult _ v = v
annotateIOError :: String -> IOError -> IO (Result a)
annotateIOError description exc =
return . Bad $ description ++ ": " ++ show exc
-- * Misc functionality
-- | Return the first result with a True condition, or the default otherwise.
select :: a -- ^ default result
-> [(Bool, a)] -- ^ list of \"condition, result\"
-> a -- ^ first result which has a True condition, or default
select def = maybe def snd . find fst
-- * Lookup of partial names functionality
-- | The priority of a match in a lookup result.
data MatchPriority = ExactMatch
| MultipleMatch
| PartialMatch
| FailMatch
deriving (Show, Read, Enum, Eq, Ord)
-- | The result of a name lookup in a list.
data LookupResult = LookupResult
{ lrMatchPriority :: MatchPriority -- ^ The result type
-- | Matching value (for ExactMatch, PartialMatch), Lookup string otherwise
, lrContent :: String
} deriving (Show, Read)
-- | Lookup results have an absolute preference ordering.
instance Eq LookupResult where
(==) = (==) `on` lrMatchPriority
instance Ord LookupResult where
compare = compare `on` lrMatchPriority
-- | Check for prefix matches in names.
-- Implemented in Ganeti core utils.text.MatchNameComponent
-- as the regexp r"^%s(\..*)?$" % re.escape(key)
prefixMatch :: String -- ^ Lookup
-> String -- ^ Full name
-> Bool -- ^ Whether there is a prefix match
prefixMatch = isPrefixOf . (++ ".")
-- | Is the lookup priority a "good" one?
goodMatchPriority :: MatchPriority -> Bool
goodMatchPriority ExactMatch = True
goodMatchPriority PartialMatch = True
goodMatchPriority _ = False
-- | Is the lookup result an actual match?
goodLookupResult :: LookupResult -> Bool
goodLookupResult = goodMatchPriority . lrMatchPriority
-- | Compares a canonical name and a lookup string.
compareNameComponent :: String -- ^ Canonical (target) name
-> String -- ^ Partial (lookup) name
-> LookupResult -- ^ Result of the lookup
compareNameComponent cnl lkp =
select (LookupResult FailMatch lkp)
[ (cnl == lkp , LookupResult ExactMatch cnl)
, (prefixMatch lkp cnl , LookupResult PartialMatch cnl)
]
-- | Lookup a string and choose the best result.
chooseLookupResult :: String -- ^ Lookup key
-> String -- ^ String to compare to the lookup key
-> LookupResult -- ^ Previous result
-> LookupResult -- ^ New result
chooseLookupResult lkp cstr old =
-- default: use class order to pick the minimum result
select (min new old)
-- special cases:
-- short circuit if the new result is an exact match
[ (lrMatchPriority new == ExactMatch, new)
-- if both are partial matches generate a multiple match
, (partial2, LookupResult MultipleMatch lkp)
] where new = compareNameComponent cstr lkp
partial2 = all ((PartialMatch==) . lrMatchPriority) [old, new]
-- | Find the canonical name for a lookup string in a list of names.
lookupName :: [String] -- ^ List of keys
-> String -- ^ Lookup string
-> LookupResult -- ^ Result of the lookup
lookupName l s = foldr (chooseLookupResult s)
(LookupResult FailMatch s) l
......@@ -73,6 +73,7 @@ $(declareIADT "ConfdRequestType"
, ("ReqClusterMaster", 'C.confdReqClusterMaster )
, ("ReqMcPipList", 'C.confdReqMcPipList )
, ("ReqInstIpsList", 'C.confdReqInstancesIpsList )
, ("ReqNodeDrbd", 'C.confdReqNodeDrbd )
])
$(makeJSONInstance ''ConfdRequestType)
......
......@@ -227,6 +227,19 @@ buildResponse cdata (ConfdRequest { confdRqType = ReqNodePipByInstPip
buildResponse _ (ConfdRequest { confdRqType = ReqNodePipByInstPip }) =
return queryArgumentError
buildResponse cdata req@(ConfdRequest { confdRqType = ReqNodeDrbd }) = do
let cfg = fst cdata
node_name <- case confdRqQuery req of
PlainQuery str -> return str
_ -> fail $ "Invalid query type " ++ show (confdRqQuery req)
node <- getNode cfg node_name
let minors = concatMap (getInstMinorsForNode (nodeName node)) .
M.elems . configInstances $ cfg
encoded = [J.JSArray [J.showJSON a, J.showJSON b, J.showJSON c,
J.showJSON d, J.showJSON e, J.showJSON f] |
(a, b, c, d, e, f) <- minors]
return (ReplyStatusOk, J.showJSON encoded)
-- | Parses a signed request.
parseRequest :: HashKey -> String -> Result (String, String, ConfdRequest)
parseRequest key str = do
......
......@@ -32,11 +32,14 @@ module Ganeti.Config
, getNode
, getInstance
, getInstPrimaryNode
, getInstMinorsForNode
, buildLinkIpInstnameMap
, instNodes
) where
import Data.List (foldl')
import qualified Data.Map as M
import qualified Data.Set as S
import qualified Text.JSON as J
import Ganeti.HTools.JSON
......@@ -62,13 +65,36 @@ loadConfig = fmap parseConfig . readConfig
-- * Query functions
-- | Computes the nodes covered by a disk.
computeDiskNodes :: Disk -> S.Set String
computeDiskNodes dsk =
case diskLogicalId dsk of
LIDDrbd8 nodeA nodeB _ _ _ _ -> S.fromList [nodeA, nodeB]
_ -> S.empty
-- | Computes all disk-related nodes of an instance. For non-DRBD,
-- this will be empty, for DRBD it will contain both the primary and
-- the secondaries.
instDiskNodes :: Instance -> S.Set String
instDiskNodes = S.unions . map computeDiskNodes . instDisks
-- | Computes all nodes of an instance.
instNodes :: Instance -> S.Set String
instNodes inst = instPrimaryNode inst `S.insert` instDiskNodes inst
-- | Computes the secondary nodes of an instance. Since this is valid
-- only for DRBD, we call directly 'instDiskNodes', skipping over the
-- extra primary insert.
instSecondaryNodes :: Instance -> S.Set String
instSecondaryNodes inst =
instPrimaryNode inst `S.delete` instDiskNodes inst
-- | Get instances of a given node.
getNodeInstances :: ConfigData -> String -> ([Instance], [Instance])
getNodeInstances cfg nname =
let all_inst = M.elems . configInstances $ cfg
pri_inst = filter ((== nname) . instPrimaryNode) all_inst
-- FIXME: actually compute the secondary nodes
sec_inst = undefined
sec_inst = filter ((nname `S.member`) . instSecondaryNodes) all_inst
in (pri_inst, sec_inst)
-- | Returns the default cluster link.
......@@ -81,23 +107,70 @@ getInstancesIpByLink :: LinkIpMap -> String -> [String]
getInstancesIpByLink linkipmap link =
M.keys $ M.findWithDefault M.empty link linkipmap
-- | Generic lookup function that converts from a possible abbreviated
-- name to a full name.
getItem :: String -> String -> M.Map String a -> Result a
getItem kind name allitems = do
let lresult = lookupName (M.keys allitems) name
err = \details -> Bad $ kind ++ " name " ++ name ++ " " ++ details
fullname <- case lrMatchPriority lresult of
PartialMatch -> Ok $ lrContent lresult
ExactMatch -> Ok $ lrContent lresult
MultipleMatch -> err "has multiple matches"
FailMatch -> err "not found"
maybe (err "not found after successfull match?!") Ok $
M.lookup fullname allitems
-- | Looks up a node.
getNode :: ConfigData -> String -> Result Node
getNode cfg name =
maybe (Bad $ "Node " ++ name ++ " not found") Ok $
M.lookup name (configNodes cfg)
getNode cfg name = getItem "Node" name (configNodes cfg)
-- | Looks up an instance.
getInstance :: ConfigData -> String -> Result Instance
getInstance cfg name =
maybe (Bad $ "Instance " ++ name ++ " not found") Ok $
M.lookup name (configInstances cfg)
getInstance cfg name = getItem "Instance" name (configInstances cfg)
-- | Looks up an instance's primary node.
getInstPrimaryNode :: ConfigData -> String -> Result Node
getInstPrimaryNode cfg name =
getInstance cfg name >>= return . instPrimaryNode >>= getNode cfg
-- | Filters DRBD minors for a given node.
getDrbdMinorsForNode :: String -> Disk -> [(Int, String)]
getDrbdMinorsForNode node disk =
let child_minors = concatMap (getDrbdMinorsForNode node) (diskChildren disk)
this_minors =
case diskLogicalId disk of
LIDDrbd8 nodeA nodeB _ minorA minorB _
| nodeA == node -> [(minorA, nodeB)]
| nodeB == node -> [(minorB, nodeA)]
_ -> []
in this_minors ++ child_minors
-- | String for primary role.
rolePrimary :: String
rolePrimary = "primary"
-- | String for secondary role.
roleSecondary :: String
roleSecondary = "secondary"
-- | Gets the list of DRBD minors for an instance that are related to
-- a given node.
getInstMinorsForNode :: String -> Instance
-> [(String, Int, String, String, String, String)]
getInstMinorsForNode node inst =
let role = if node == instPrimaryNode inst
then rolePrimary
else roleSecondary
iname = instName inst
-- FIXME: the disk/ build there is hack-ish; unify this in a
-- separate place, or reuse the iv_name (but that is deprecated on
-- the Python side)
in concatMap (\(idx, dsk) ->
[(node, minor, iname, "disk/" ++ show idx, role, peer)
| (minor, peer) <- getDrbdMinorsForNode node dsk]) .
zip [(0::Int)..] . instDisks $ inst
-- | Builds link -> ip -> instname map.
--
-- TODO: improve this by splitting it into multiple independent functions:
......
......@@ -100,7 +100,7 @@ import qualified Ganeti.HTools.Node as Node
import qualified Ganeti.Constants as C
import Ganeti.HTools.Types
import Ganeti.HTools.Utils
import Ganeti.HTools.Loader
import Ganeti.BasicTypes
-- * Constants
......
......@@ -30,8 +30,6 @@ module Ganeti.HTools.Loader
( mergeData
, checkData
, assignIndices
, lookupName
, goodLookupResult
, lookupNode
, lookupInstance
, lookupGroup
......@@ -40,14 +38,9 @@ module Ganeti.HTools.Loader
, Request(..)
, ClusterData(..)
, emptyCluster
, compareNameComponent
, prefixMatch
, LookupResult(..)
, MatchPriority(..)
) where
import Data.List
import Data.Function
import qualified Data.Map as M
import Text.Printf (printf)
......@@ -57,6 +50,7 @@ import qualified Ganeti.HTools.Node as Node
import qualified Ganeti.HTools.Group as Group
import qualified Ganeti.HTools.Cluster as Cluster
import Ganeti.BasicTypes
import Ganeti.HTools.Types
import Ganeti.HTools.Utils
......@@ -94,27 +88,6 @@ data ClusterData = ClusterData
, cdIPolicy :: IPolicy -- ^ The cluster instance policy
} deriving (Show, Read, Eq)
-- | The priority of a match in a lookup result.
data MatchPriority = ExactMatch
| MultipleMatch
| PartialMatch
| FailMatch
deriving (Show, Read, Enum, Eq, Ord)
-- | The result of a name lookup in a list.
data LookupResult = LookupResult
{ lrMatchPriority :: MatchPriority -- ^ The result type
-- | Matching value (for ExactMatch, PartialMatch), Lookup string otherwise
, lrContent :: String
} deriving (Show, Read)
-- | Lookup results have an absolute preference ordering.
instance Eq LookupResult where
(==) = (==) `on` lrMatchPriority
instance Ord LookupResult where
compare = compare `on` lrMatchPriority
-- | An empty cluster.
emptyCluster :: ClusterData
emptyCluster = ClusterData Container.empty Container.empty Container.empty []
......@@ -139,57 +112,6 @@ lookupGroup ktg nname gname =
maybe (fail $ "Unknown group '" ++ gname ++ "' for node " ++ nname) return $
M.lookup gname ktg
-- | Check for prefix matches in names.
-- Implemented in Ganeti core utils.text.MatchNameComponent
-- as the regexp r"^%s(\..*)?$" % re.escape(key)
prefixMatch :: String -- ^ Lookup
-> String -- ^ Full name
-> Bool -- ^ Whether there is a prefix match
prefixMatch = isPrefixOf . (++ ".")
-- | Is the lookup priority a "good" one?
goodMatchPriority :: MatchPriority -> Bool
goodMatchPriority ExactMatch = True
goodMatchPriority PartialMatch = True
goodMatchPriority _ = False
-- | Is the lookup result an actual match?
goodLookupResult :: LookupResult -> Bool
goodLookupResult = goodMatchPriority . lrMatchPriority
-- | Compares a canonical name and a lookup string.
compareNameComponent :: String -- ^ Canonical (target) name
-> String -- ^ Partial (lookup) name
-> LookupResult -- ^ Result of the lookup
compareNameComponent cnl lkp =
select (LookupResult FailMatch lkp)
[ (cnl == lkp , LookupResult ExactMatch cnl)
, (prefixMatch lkp cnl , LookupResult PartialMatch cnl)
]
-- | Lookup a string and choose the best result.
chooseLookupResult :: String -- ^ Lookup key
-> String -- ^ String to compare to the lookup key
-> LookupResult -- ^ Previous result
-> LookupResult -- ^ New result
chooseLookupResult lkp cstr old =
-- default: use class order to pick the minimum result
select (min new old)
-- special cases:
-- short circuit if the new result is an exact match
[ (lrMatchPriority new == ExactMatch, new)
-- if both are partial matches generate a multiple match
, (partial2, LookupResult MultipleMatch lkp)
] where new = compareNameComponent cstr lkp
partial2 = all ((PartialMatch==) . lrMatchPriority) [old, new]
-- | Find the canonical name for a lookup string in a list of names.
lookupName :: [String] -- ^ List of keys
-> String -- ^ Lookup string
-> LookupResult -- ^ Result of the lookup
lookupName l s = foldr (chooseLookupResult s)
(LookupResult FailMatch s) l
-- | Given a list of elements (and their names), assign indices to them.
assignIndices :: (Element a) =>
[(String, a)]
......
......@@ -65,6 +65,7 @@ import qualified Ganeti.Confd as Confd
import qualified Ganeti.Config as Config
import qualified Ganeti.Daemon as Daemon
import qualified Ganeti.Hash as Hash
import qualified Ganeti.BasicTypes as BasicTypes
import qualified Ganeti.Jobs as Jobs
import qualified Ganeti.Logging as Logging
import qualified Ganeti.Luxi as Luxi
......@@ -1592,14 +1593,14 @@ prop_Loader_mergeData ns =
-- | Check that compareNameComponent on equal strings works.
prop_Loader_compareNameComponent_equal :: String -> Bool
prop_Loader_compareNameComponent_equal s =
Loader.compareNameComponent s s ==
Loader.LookupResult Loader.ExactMatch s
BasicTypes.compareNameComponent s s ==
BasicTypes.LookupResult BasicTypes.ExactMatch s
-- | Check that compareNameComponent on prefix strings works.
prop_Loader_compareNameComponent_prefix :: NonEmptyList Char -> String -> Bool
prop_Loader_compareNameComponent_prefix (NonEmpty s1) s2 =
Loader.compareNameComponent (s1 ++ "." ++ s2) s1 ==
Loader.LookupResult Loader.PartialMatch s1
BasicTypes.compareNameComponent (s1 ++ "." ++ s2) s1 ==
BasicTypes.LookupResult BasicTypes.PartialMatch s1
testSuite "Loader"
[ 'prop_Loader_lookupNode
......
......@@ -132,13 +132,6 @@ if' :: Bool -- ^ condition
if' True x _ = x
if' _ _ y = y
-- | Return the first result with a True condition, or the default otherwise.
select :: a -- ^ default result
-> [(Bool, a)] -- ^ list of \"condition, result\"
-> a -- ^ first result which has a True condition, or default
select def = maybe def snd . find fst
-- * Parsing utility functions
-- | Parse results from readsPrec.
......
......@@ -36,6 +36,7 @@ module Ganeti.Objects
, PartialNIC(..)
, DiskMode(..)
, DiskType(..)
, DiskLogicalId(..)
, Disk(..)
, DiskTemplate(..)
, PartialBEParams(..)
......@@ -54,7 +55,8 @@ module Ganeti.Objects
) where
import Data.Maybe
import Text.JSON (makeObj, showJSON, readJSON)
import Text.JSON (makeObj, showJSON, readJSON, JSON, JSValue(..))
import qualified Text.JSON as J
import qualified Ganeti.Constants as C
import Ganeti.HTools.JSON
......@@ -93,16 +95,119 @@ $(declareSADT "DiskType"
, ("LD_DRBD8", 'C.ldDrbd8)
, ("LD_FILE", 'C.ldFile)
, ("LD_BLOCKDEV", 'C.ldBlockdev)
, ("LD_RADOS", 'C.ldRbd)
])
$(makeJSONInstance ''DiskType)
-- | The file driver type.
$(declareSADT "FileDriver"
[ ("FileLoop", 'C.fdLoop)
, ("FileBlktap", 'C.fdBlktap)
])
$(makeJSONInstance ''FileDriver)
-- | The persistent block driver type. Currently only one type is allowed.
$(declareSADT "BlockDriver"
[ ("BlockDrvManual", 'C.blockdevDriverManual)
])
$(makeJSONInstance ''BlockDriver)
-- | Constant for the dev_type key entry in the disk config.
devType :: String
devType = "dev_type"
-- | The disk configuration type. This includes the disk type itself,
-- for a more complete consistency. Note that since in the Python
-- code-base there's no authoritative place where we document the
-- logical id, this is probably a good reference point.
data DiskLogicalId
= LIDPlain String String -- ^ Volume group, logical volume
| LIDDrbd8 String String Int Int Int String
-- ^ NodeA, NodeB, Port, MinorA, MinorB, Secret
| LIDFile FileDriver String -- ^ Driver, path
| LIDBlockDev BlockDriver String -- ^ Driver, path (must be under /dev)
| LIDRados String String -- ^ Unused, path
deriving (Read, Show, Eq)
-- | Mapping from a logical id to a disk type.
lidDiskType :: DiskLogicalId -> DiskType
lidDiskType (LIDPlain {}) = LD_LV
lidDiskType (LIDDrbd8 {}) = LD_DRBD8
lidDiskType (LIDFile {}) = LD_FILE
lidDiskType (LIDBlockDev {}) = LD_BLOCKDEV
lidDiskType (LIDRados {}) = LD_RADOS
-- | Builds the extra disk_type field for a given logical id.
lidEncodeType :: DiskLogicalId -> [(String, JSValue)]
lidEncodeType v = [(devType, showJSON . lidDiskType $ v)]
-- | Custom encoder for DiskLogicalId (logical id only).
encodeDLId :: DiskLogicalId -> JSValue
encodeDLId (LIDPlain vg lv) = JSArray [showJSON vg, showJSON lv]
encodeDLId (LIDDrbd8 nodeA nodeB port minorA minorB key) =
JSArray [ showJSON nodeA, showJSON nodeB, showJSON port
, showJSON minorA, showJSON minorB, showJSON key ]
encodeDLId (LIDRados pool name) = JSArray [showJSON pool, showJSON name]
encodeDLId (LIDFile driver name) = JSArray [showJSON driver, showJSON name]
encodeDLId (LIDBlockDev driver name) = JSArray [showJSON driver, showJSON name]
-- | Custom encoder for DiskLogicalId, composing both the logical id
-- and the extra disk_type field.
encodeFullDLId :: DiskLogicalId -> (JSValue, [(String, JSValue)])
encodeFullDLId v = (encodeDLId v, lidEncodeType v)
-- | Custom decoder for DiskLogicalId. This is manual for now, since
-- we don't have yet automation for separate-key style fields.
decodeDLId :: [(String, JSValue)] -> JSValue -> J.Result DiskLogicalId
decodeDLId obj lid = do
dtype <- fromObj obj devType
case dtype of
LD_DRBD8 ->
case lid of
JSArray [nA, nB, p, mA, mB, k] -> do
nA' <- readJSON nA
nB' <- readJSON nB
p' <- readJSON p
mA' <- readJSON mA
mB' <- readJSON mB
k' <- readJSON k
return $ LIDDrbd8 nA' nB' p' mA' mB' k'
_ -> fail $ "Can't read logical_id for DRBD8 type"
LD_LV ->
case lid of
JSArray [vg, lv] -> do
vg' <- readJSON vg
lv' <- readJSON lv
return $ LIDPlain vg' lv'
_ -> fail $ "Can't read logical_id for plain type"
LD_FILE ->
case lid of