Commit c79d19e8 authored by Klaus Aehlig's avatar Klaus Aehlig

Merge branch 'stable-2.11' into stable-2.12

* stable-2.11
  (no changes)

* stable-2.10
  Use more efficient statistics for the standard deviation
  Use statistics updates when allocating on pairs
  Factor score computation through abstract statistics
  Verify the update of the standard deviation statistics
  Add data type for abstract statistics
  Relax test requirements
  Fix gnt-network client wrt instances report
  Fix QueryNetworks wrt instances

Conflicts:
	src/Ganeti/HTools/Cluster.hs: manually apply
	    patch b366d732 to stable-2.11 version
	src/Ganeti/Utils.hs: take all addition
Signed-off-by: default avatarKlaus Aehlig <aehlig@google.com>
Reviewed-by: default avatarPetr Pudlak <pudlak@google.com>
parents 1d425eb9 c3a9cd35
......@@ -246,12 +246,8 @@ def ShowNetworkConfig(_, args):
ToStdout(" not connected to any node group")
if instances:
idata = cl.QueryInstances([], ["uuid", "name"], False)
uuid2name = dict(idata)
ToStdout(" used by %d instances:", len(instances))
for inst in instances:
name = uuid2name[inst]
for name in instances:
((ips, networks), ) = cl.QueryInstances([name],
["nic.ips", "nic.networks"],
use_locking=False)
......
......@@ -319,70 +319,107 @@ computeAllocationDelta cini cfin =
}
in (rini, rfin, runa)
-- | The names and weights of the individual elements in the CV list, together
-- with their statistical accumulation function and a bit to decide whether it
-- is a statistics for online nodes.
detailedCVInfoExt :: [((Double, String), ([Double] -> Statistics, Bool))]
detailedCVInfoExt = [ ((1, "free_mem_cv"), (getStdDevStatistics, True))
, ((1, "free_disk_cv"), (getStdDevStatistics, True))
, ((1, "n1_cnt"), (getSumStatistics, True))
, ((1, "reserved_mem_cv"), (getStdDevStatistics, True))
, ((4, "offline_all_cnt"), (getSumStatistics, False))
, ((16, "offline_pri_cnt"), (getSumStatistics, False))
, ((1, "vcpu_ratio_cv"), (getStdDevStatistics, True))
, ((1, "cpu_load_cv"), (getStdDevStatistics, True))
, ((1, "mem_load_cv"), (getStdDevStatistics, True))
, ((1, "disk_load_cv"), (getStdDevStatistics, True))
, ((1, "net_load_cv"), (getStdDevStatistics, True))
, ((2, "pri_tags_score"), (getSumStatistics, True))
, ((1, "spindles_cv"), (getStdDevStatistics, True))
]
-- | The names and weights of the individual elements in the CV list.
detailedCVInfo :: [(Double, String)]
detailedCVInfo = [ (1, "free_mem_cv")
, (1, "free_disk_cv")
, (1, "n1_cnt")
, (1, "reserved_mem_cv")
, (4, "offline_all_cnt")
, (16, "offline_pri_cnt")
, (1, "vcpu_ratio_cv")
, (1, "cpu_load_cv")
, (1, "mem_load_cv")
, (1, "disk_load_cv")
, (1, "net_load_cv")
, (2, "pri_tags_score")
, (1, "spindles_cv")
]
detailedCVInfo = map fst detailedCVInfoExt
-- | Holds the weights used by 'compCVNodes' for each metric.
detailedCVWeights :: [Double]
detailedCVWeights = map fst detailedCVInfo
-- | Compute the mem and disk covariance.
compDetailedCV :: [Node.Node] -> [Double]
compDetailedCV all_nodes =
-- | The aggregation functions for the weights
detailedCVAggregation :: [([Double] -> Statistics, Bool)]
detailedCVAggregation = map snd detailedCVInfoExt
-- | The bit vector describing which parts of the statistics are
-- for online nodes.
detailedCVOnlineStatus :: [Bool]
detailedCVOnlineStatus = map snd detailedCVAggregation
-- | Compute statistical measures of a single node.
compDetailedCVNode :: Node.Node -> [Double]
compDetailedCVNode node =
let mem = Node.pMem node
dsk = Node.pDsk node
n1 = fromIntegral
$ if Node.failN1 node
then length (Node.sList node) + length (Node.pList node)
else 0
res = Node.pRem node
ipri = fromIntegral . length $ Node.pList node
isec = fromIntegral . length $ Node.sList node
ioff = ipri + isec
cpu = Node.pCpuEff node
DynUtil c1 m1 d1 nn1 = Node.utilLoad node
DynUtil c2 m2 d2 nn2 = Node.utilPool node
(c_load, m_load, d_load, n_load) = (c1/c2, m1/m2, d1/d2, nn1/nn2)
pri_tags = fromIntegral $ Node.conflictingPrimaries node
spindles = Node.instSpindles node / Node.hiSpindles node
in [ mem, dsk, n1, res, ioff, ipri, cpu
, c_load, m_load, d_load, n_load
, pri_tags, spindles
]
-- | Compute the statistics of a cluster.
compClusterStatistics :: [Node.Node] -> [Statistics]
compClusterStatistics all_nodes =
let (offline, nodes) = partition Node.offline all_nodes
mem_l = map Node.pMem nodes
dsk_l = map Node.pDsk nodes
-- metric: memory covariance
mem_cv = stdDev mem_l
-- metric: disk covariance
dsk_cv = stdDev dsk_l
-- metric: count of instances living on N1 failing nodes
n1_score = fromIntegral . sum . map (\n -> length (Node.sList n) +
length (Node.pList n)) .
filter Node.failN1 $ nodes :: Double
res_l = map Node.pRem nodes
-- metric: reserved memory covariance
res_cv = stdDev res_l
-- offline instances metrics
offline_ipri = sum . map (length . Node.pList) $ offline
offline_isec = sum . map (length . Node.sList) $ offline
-- metric: count of instances on offline nodes
off_score = fromIntegral (offline_ipri + offline_isec)::Double
-- metric: count of primary instances on offline nodes (this
-- helps with evacuation/failover of primary instances on
-- 2-node clusters with one node offline)
off_pri_score = fromIntegral offline_ipri::Double
cpu_l = map Node.pCpuEff nodes
-- metric: covariance of effective vcpu/pcpu ratio
cpu_cv = stdDev cpu_l
-- metrics: covariance of cpu, memory, disk and network load
(c_load, m_load, d_load, n_load) =
unzip4 $ map (\n ->
let DynUtil c1 m1 d1 n1 = Node.utilLoad n
DynUtil c2 m2 d2 n2 = Node.utilPool n
in (c1/c2, m1/m2, d1/d2, n1/n2)) nodes
-- metric: conflicting instance count
pri_tags_inst = sum $ map Node.conflictingPrimaries nodes
pri_tags_score = fromIntegral pri_tags_inst::Double
-- metric: spindles %
spindles_cv = map (\n -> Node.instSpindles n / Node.hiSpindles n) nodes
in [ mem_cv, dsk_cv, n1_score, res_cv, off_score, off_pri_score, cpu_cv
, stdDev c_load, stdDev m_load , stdDev d_load, stdDev n_load
, pri_tags_score, stdDev spindles_cv ]
offline_values = transpose (map compDetailedCVNode offline)
++ repeat []
-- transpose of an empty list is empty and not k times the empty list, as
-- would be the transpose of a 0 x k matrix
online_values = transpose $ map compDetailedCVNode nodes
aggregate (f, True) (onNodes, _) = f onNodes
aggregate (f, False) (_, offNodes) = f offNodes
in zipWith aggregate detailedCVAggregation
$ zip online_values offline_values
-- | Update a cluster statistics by replacing the contribution of one
-- node by that of another.
updateClusterStatistics :: [Statistics]
-> (Node.Node, Node.Node) -> [Statistics]
updateClusterStatistics stats (old, new) =
let update = zip (compDetailedCVNode old) (compDetailedCVNode new)
online = not $ Node.offline old
updateStat forOnline stat upd = if forOnline == online
then updateStatistics stat upd
else stat
in zipWith3 updateStat detailedCVOnlineStatus stats update
-- | Update a cluster statistics twice.
updateClusterStatisticsTwice :: [Statistics]
-> (Node.Node, Node.Node)
-> (Node.Node, Node.Node)
-> [Statistics]
updateClusterStatisticsTwice s a =
updateClusterStatistics (updateClusterStatistics s a)
-- | Compute cluster statistics
compDetailedCV :: [Node.Node] -> [Double]
compDetailedCV = map getStatisticValue . compClusterStatistics
-- | Compute the cluster score from its statistics
compCVfromStats :: [Statistics] -> Double
compCVfromStats = sum . zipWith (*) detailedCVWeights . map getStatisticValue
-- | Compute the /total/ variance.
compCVNodes :: [Node.Node] -> Double
......@@ -511,9 +548,10 @@ allocateOnSingle nl inst new_pdx =
return (new_nl, new_inst, [new_p], new_score)
-- | Tries to allocate an instance on a given pair of nodes.
allocateOnPair :: Node.List -> Instance.Instance -> Ndx -> Ndx
allocateOnPair :: [Statistics]
-> Node.List -> Instance.Instance -> Ndx -> Ndx
-> OpResult Node.AllocElement
allocateOnPair nl inst new_pdx new_sdx =
allocateOnPair stats nl inst new_pdx new_sdx =
let tgt_p = Container.find new_pdx nl
tgt_s = Container.find new_sdx nl
in do
......@@ -523,7 +561,9 @@ allocateOnPair nl inst new_pdx new_sdx =
new_s <- Node.addSec tgt_s inst new_pdx
let new_inst = Instance.setBoth inst new_pdx new_sdx
new_nl = Container.addTwo new_pdx new_p new_sdx new_s nl
return (new_nl, new_inst, [new_p, new_s], compCV new_nl)
new_stats = updateClusterStatisticsTwice stats
(tgt_p, new_p) (tgt_s, new_s)
return (new_nl, new_inst, [new_p, new_s], compCVfromStats new_stats)
-- | Tries to perform an instance move and returns the best table
-- between the original one and the new one.
......@@ -794,10 +834,11 @@ tryAlloc :: (Monad m) =>
-> m AllocSolution -- ^ Possible solution list
tryAlloc _ _ _ (Right []) = fail "Not enough online nodes"
tryAlloc nl _ inst (Right ok_pairs) =
let psols = parMap rwhnf (\(p, ss) ->
let cstat = compClusterStatistics $ Container.elems nl
psols = parMap rwhnf (\(p, ss) ->
foldl' (\cstate ->
concatAllocs cstate .
allocateOnPair nl inst p)
allocateOnPair cstat nl inst p)
emptyAllocSolution ss) ok_pairs
sols = foldl' sumAllocs emptyAllocSolution psols
in return $ annotateSolution sols
......
......@@ -131,14 +131,13 @@ getNicLink nic_params = fromMaybe "-" (nicpLinkP nic_params)
-- | Retrieves the network's instances' names.
getInstances :: ConfigData -> String -> [String]
getInstances cfg network_uuid =
map instName (filter (instIsConnected cfg network_uuid)
map instName (filter (instIsConnected network_uuid)
((Map.elems . fromContainer . configInstances) cfg))
-- | Helper function that checks if an instance is linked to the given network.
instIsConnected :: ConfigData -> String -> Instance -> Bool
instIsConnected cfg network_uuid inst =
network_uuid `elem` mapMaybe (getNetworkUuid cfg)
(mapMaybe nicNetwork (instNics inst))
instIsConnected :: String -> Instance -> Bool
instIsConnected network_uuid inst =
network_uuid `elem` mapMaybe nicNetwork (instNics inst)
-- | Helper function to look up a network's UUID by its name
getNetworkUuid :: ConfigData -> String -> Maybe String
......
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleContexts, BangPatterns #-}
{-| Utility functions. -}
......@@ -29,6 +29,11 @@ module Ganeti.Utils
, debugXy
, sepSplit
, findFirst
, Statistics
, getSumStatistics
, getStdDevStatistics
, getStatisticValue
, updateStatistics
, stdDev
, if'
, select
......@@ -189,6 +194,48 @@ stdDev lst =
av = foldl' (\accu em -> let d = em - mv in accu + d * d) 0.0 lst
in sqrt (av / ll) -- stddev
-- | Abstract type of statistical accumulations. They behave as if the given
-- statistics were computed on the list of values, but they allow a potentially
-- more efficient update of a given value.
data Statistics = SumStatistics Double
| StdDevStatistics Double Double Double deriving Show
-- count, sum, and not the sum of squares---instead the
-- computed variance for better precission.
-- | Get a statistics that sums up the values.
getSumStatistics :: [Double] -> Statistics
getSumStatistics = SumStatistics . sum
-- | Get a statistics for the standard deviation.
getStdDevStatistics :: [Double] -> Statistics
getStdDevStatistics xs =
let (nt, st) = foldl' (\(n, s) x ->
let !n' = n + 1
!s' = s + x
in (n', s'))
(0, 0) xs
mean = st / nt
nvar = foldl' (\v x -> let d = x - mean in v + d * d) 0 xs
in StdDevStatistics nt st (nvar / nt)
-- | Obtain the value of a statistics.
getStatisticValue :: Statistics -> Double
getStatisticValue (SumStatistics s) = s
getStatisticValue (StdDevStatistics _ _ var) = sqrt var
-- | In a given statistics replace on value by another. This
-- will only give meaningful results, if the original value
-- was actually part of the statistics.
updateStatistics :: Statistics -> (Double, Double) -> Statistics
updateStatistics (SumStatistics s) (x, y) = SumStatistics $ s + (y - x)
updateStatistics (StdDevStatistics n s var) (x, y) =
let !ds = y - x
!dss = y * y - x * x
!dnnvar = n * dss - (2 * s + ds) * ds
!s' = s + ds
!var' = max 0 $ var + dnnvar / (n * n)
in StdDevStatistics n s' var'
-- * Logical functions
-- Avoid syntactic sugar and enhance readability. These functions are proposed
......
......@@ -68,7 +68,9 @@ isNodeBig size node = Node.availDisk node > size * Types.unitDsk
&& Node.availCpu node > size * Types.unitCpu
canBalance :: Cluster.Table -> Bool -> Bool -> Bool -> Bool
canBalance tbl dm im evac = isJust $ Cluster.tryBalance tbl dm im evac False 0 0
canBalance tbl@(Cluster.Table _ _ ini_cv _) dm im evac =
maybe False (\(Cluster.Table _ _ fin_cv _) -> ini_cv - fin_cv > 1e-12)
$ Cluster.tryBalance tbl dm im evac False 0 0
-- | Assigns a new fresh instance to a cluster; this is not
-- allocation, so no resource checks are done.
......@@ -147,8 +149,8 @@ prop_Alloc_sane inst =
tbl = Cluster.Table xnl il' cv []
in printTestCase "Cluster can be balanced after allocation"
(not (canBalance tbl True True False)) .&&.
printTestCase "Solution score differs from actual node list:"
(Cluster.compCV xnl ==? cv)
printTestCase "Solution score differs from actual node list"
(abs (Cluster.compCV xnl - cv) < 1e-12)
-- | Checks that on a 2-5 node cluster, we can allocate a random
-- instance spec via tiered allocation (whatever the original instance
......
......@@ -170,7 +170,7 @@ enhanceInstWithNets inst nets = do
uuid <- arbitrary
-- generate some more networks than the given ones
num_more_nets <- choose (0,3)
more_nets <- vectorOf num_more_nets genName
more_nets <- vectorOf num_more_nets genUUID
let genNic net = PartialNic mac ip nicparams net name uuid
partial_nics = map (genNic . Just)
(List.nub (nets ++ more_nets))
......
......@@ -33,7 +33,6 @@ module Test.Ganeti.Query.Network
import Ganeti.JSON
import Ganeti.Objects
import Ganeti.Query.Network
import Ganeti.Types
import Test.Ganeti.Objects
import Test.Ganeti.TestCommon
......@@ -67,9 +66,8 @@ prop_instIsConnected :: ConfigData -> Property
prop_instIsConnected cfg =
let nets = (fromContainer . configNetworks) cfg
net_keys = Map.keys nets
net_names = map (fromNonEmpty . networkName) (Map.elems nets)
in forAll (genInstWithNets net_names) $ \inst ->
True ==? all (\nk -> instIsConnected cfg nk inst) net_keys
in forAll (genInstWithNets net_keys) $ \inst ->
True ==? all (`instIsConnected` inst) net_keys
-- | Tests whether instances that are not connected to a network are
-- correctly classified as such.
......@@ -77,10 +75,9 @@ prop_instIsConnected_notFound :: ConfigData -> String -> Property
prop_instIsConnected_notFound cfg network_uuid =
let nets = (fromContainer . configNetworks) cfg
net_keys = Map.keys nets
net_names = map (fromNonEmpty . networkName) (Map.elems nets)
in notElem network_uuid net_keys ==>
forAll (genInstWithNets net_names) $ \inst ->
not (instIsConnected cfg network_uuid inst)
forAll (genInstWithNets net_keys) $ \inst ->
not (instIsConnected network_uuid inst)
testSuite "Query_Network"
[ 'prop_getGroupConnection
......
......@@ -341,6 +341,23 @@ prop_splitRecombineEithers es =
(splitleft, splitright, trail) = splitEithers es
emptylist = []::[Int]
-- | Test the update function for standard deviations against the naive
-- implementation.
prop_stddev_update :: Property
prop_stddev_update =
forAll (choose (0, 6) >>= flip vectorOf (choose (0, 1))) $ \xs ->
forAll (choose (0, 1)) $ \a ->
forAll (choose (0, 1)) $ \b ->
forAll (choose (1, 6) >>= flip vectorOf (choose (0, 1))) $ \ys ->
let original = xs ++ [a] ++ ys
modified = xs ++ [b] ++ ys
with_update = getStatisticValue
$ updateStatistics (getStdDevStatistics original) (a,b)
direct = stdDev modified
in printTestCase ("Value computed by update " ++ show with_update
++ " differs too much from correct value " ++ show direct)
(abs (with_update - direct) < 1e-12)
-- | Test list for the Utils module.
testSuite "Utils"
[ 'prop_commaJoinSplit
......@@ -368,4 +385,5 @@ testSuite "Utils"
, 'prop_chompPrefix_empty_string
, 'prop_chompPrefix_nothing
, 'prop_splitRecombineEithers
, 'prop_stddev_update
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment