Add loading and processing of utilisation data

This patch adds loading and processing the utilisation data during
instance moves. While the data is not yet used, it is correctly modified
by instance changes between nodes.

hbal has the new ‘-U’ command line argument for this. The format of the
file is simply instance name and the four stats, space-separated.

Ganeti/HTools/ExtLoader.hs

@@ -52,6 +52,7 @@ import qualified Ganeti.HTools.Node as Node
 
 import Ganeti.HTools.Types
 import Ganeti.HTools.CLI
+import Ganeti.HTools.Utils (sepSplit, tryRead)
 
 -- | Parse the environment and return the node\/instance names.
 --
@@ -66,6 +67,20 @@ parseEnv () = do
 wrapIO :: IO (Result a) -> IO (Result a)
 wrapIO = flip catch (return . Bad . show)
 
+parseUtilisation :: String -> Result (String, DynUtil)
+parseUtilisation line =
+    let columns = sepSplit ' ' line
+    in case columns of
+         [name, cpu, mem, dsk, net] -> do
+                      rcpu <- tryRead name cpu
+                      rmem <- tryRead name mem
+                      rdsk <- tryRead name dsk
+                      rnet <- tryRead name net
+                      let du = DynUtil { cpuWeight = rcpu, memWeight = rmem
+                                       , dskWeight = rdsk, netWeight = rnet }
+                      return (name, du)
+         _ -> Bad $ "Cannot parse line " ++ line
+
 -- | External tool data loader from a variety of sources.
 loadExternalData :: Options
                  -> IO (Node.List, Instance.List, String)
@@ -89,6 +104,16 @@ loadExternalData opts = do
                            " files options should be given.")
          exitWith $ ExitFailure 1
 
+  util_contents <- (case optDynuFile opts of
+                      Just path -> readFile path
+                      Nothing -> return "")
+  let util_data = mapM parseUtilisation $ lines util_contents
+  util_data' <- (case util_data of
+                   Ok x -> return x
+                   Bad y -> do
+                     hPutStrLn stderr ("Error: can't parse utilisation" ++
+                                       " data: " ++ show y)
+                     exitWith $ ExitFailure 1)
   input_data <-
       case () of
         _ | setRapi ->
@@ -101,7 +126,7 @@ loadExternalData opts = do
           | setSim -> Simu.loadData $ fromJust simdata
           | otherwise -> wrapIO $ Text.loadData nodef instf
 
-  let ldresult = input_data >>= Loader.mergeData
+  let ldresult = input_data >>= Loader.mergeData util_data'
   (loaded_nl, il, csf) <-
       (case ldresult of
          Ok x -> return x

Ganeti/HTools/IAlloc.hs

@@ -110,7 +110,7 @@ parseData body = do
   let idata = fromJSObject ilist
   iobj <- mapM (\(x,y) -> asJSObject y >>= parseInstance ktn x) idata
   let (kti, il) = assignIndices iobj
-  (map_n, map_i, csf) <- mergeData (nl, il)
+  (map_n, map_i, csf) <- mergeData [] (nl, il)
   req_nodes <- fromObj "required_nodes" request
   optype <- fromObj "type" request
   rqtype <-

Ganeti/HTools/Loader.hs

@@ -37,6 +37,7 @@ module Ganeti.HTools.Loader
     , Request(..)
     ) where
 
+import Control.Monad (foldM)
 import Data.Function (on)
 import Data.List
 import Data.Maybe (fromJust)
@@ -96,9 +97,9 @@ assocEqual = (==) `on` fst
 
 -- | For each instance, add its index to its primary and secondary nodes.
 fixNodes :: [(Ndx, Node.Node)]
-         -> (Idx, Instance.Instance)
+         -> Instance.Instance
          -> [(Ndx, Node.Node)]
-fixNodes accu (_, inst) =
+fixNodes accu inst =
     let
         pdx = Instance.pNode inst
         sdx = Instance.sNode inst
@@ -130,15 +131,21 @@ stripSuffix sflen name = take (length name - sflen) name
 
 -- | Initializer function that loads the data from a node and instance
 -- list and massages it into the correct format.
-mergeData :: (Node.AssocList,
+mergeData :: [(String, DynUtil)]  -- ^ Instance utilisation data
+          -> (Node.AssocList,
               Instance.AssocList) -- ^ Data from either Text.loadData
                                   -- or Rapi.loadData
           -> Result (Node.List, Instance.List, String)
-mergeData (nl, il) = do
-  let
-      nl2 = foldl' fixNodes nl il
-      il3 = Container.fromAssocList il
-      nl3 = Container.fromAssocList
+mergeData um (nl, il) = do
+  let il2 = Container.fromAssocList il
+  il3 <- foldM (\im (name, n_util) -> do
+                  idx <- Container.findByName im name
+                  let inst = Container.find idx im
+                      new_i = inst { Instance.util = n_util }
+                  return $ Container.add idx new_i im
+               ) il2 um
+  let nl2 = foldl' fixNodes nl (Container.elems il3)
+  let nl3 = Container.fromAssocList
             (map (\ (k, v) -> (k, Node.buildPeers v il3)) nl2)
       node_names = map Node.name $ Container.elems nl3
       inst_names = map Instance.name $ Container.elems il3

Ganeti/HTools/Node.hs

@@ -26,13 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 -}
 
 module Ganeti.HTools.Node
-    ( Node(failN1, name, idx,
-           tMem, nMem, fMem, rMem, xMem,
-           tDsk, fDsk,
-           tCpu, uCpu,
-           pMem, pDsk, pRem, pCpu,
-           mDsk, mCpu, loDsk, hiCpu,
-           pList, sList, offline)
+    ( Node(..)
     , List
     -- * Constructor
     , create
@@ -217,16 +211,22 @@ buildPeers t il =
     in t {peers=pmap, failN1 = new_failN1, rMem = new_rmem, pRem = new_prem}
 
 -- | Assigns an instance to a node as primary and update the used VCPU
--- count.
+-- count and utilisation data.
 setPri :: Node -> Instance.Instance -> Node
-setPri t inst = t { pList = (Instance.idx inst):pList t
+setPri t inst = t { pList = Instance.idx inst:pList t
                   , uCpu = new_count
-                  , pCpu = fromIntegral new_count / tCpu t }
+                  , pCpu = fromIntegral new_count / tCpu t
+                  , utilLoad = utilLoad t `T.addUtil` Instance.util inst
+                  }
     where new_count = uCpu t + Instance.vcpus inst
 
 -- | Assigns an instance to a node as secondary without other updates.
 setSec :: Node -> Instance.Instance -> Node
-setSec t inst = t { sList = (Instance.idx inst):sList t }
+setSec t inst = t { sList = Instance.idx inst:sList t
+                  , utilLoad = old_load { T.dskWeight = T.dskWeight old_load +
+                                          T.dskWeight (Instance.util inst) }
+                  }
+    where old_load = utilLoad t
 
 -- * Update functions
 
@@ -250,9 +250,10 @@ removePri t inst =
         new_failn1 = new_mem <= rMem t
         new_ucpu = uCpu t - Instance.vcpus inst
         new_rcpu = fromIntegral new_ucpu / tCpu t
+        new_load = utilLoad t `T.subUtil` Instance.util inst
     in t {pList = new_plist, fMem = new_mem, fDsk = new_dsk,
           failN1 = new_failn1, pMem = new_mp, pDsk = new_dp,
-          uCpu = new_ucpu, pCpu = new_rcpu}
+          uCpu = new_ucpu, pCpu = new_rcpu, utilLoad = new_load}
 
 -- | Removes a secondary instance.
 removeSec :: Node -> Instance.Instance -> Node
@@ -273,9 +274,12 @@ removeSec t inst =
         new_prem = fromIntegral new_rmem / tMem t
         new_failn1 = fMem t <= new_rmem
         new_dp = fromIntegral new_dsk / tDsk t
+        old_load = utilLoad t
+        new_load = old_load { T.dskWeight = T.dskWeight old_load -
+                                            T.dskWeight (Instance.util inst) }
     in t {sList = new_slist, fDsk = new_dsk, peers = new_peers,
           failN1 = new_failn1, rMem = new_rmem, pDsk = new_dp,
-          pRem = new_prem}
+          pRem = new_prem, utilLoad = new_load}
 
 -- | Adds a primary instance.
 addPri :: Node -> Instance.Instance -> T.OpResult Node
@@ -288,6 +292,7 @@ addPri t inst =
         new_pcpu = fromIntegral new_ucpu / tCpu t
         new_dp = fromIntegral new_dsk / tDsk t
         l_cpu = mCpu t
+        new_load = utilLoad t `T.addUtil` Instance.util inst
     in if new_mem <= 0 then T.OpFail T.FailMem
        else if new_dsk <= 0 || mDsk t > new_dp then T.OpFail T.FailDisk
        else if new_failn1 && not (failN1 t) then T.OpFail T.FailMem
@@ -297,7 +302,7 @@ addPri t inst =
                new_mp = fromIntegral new_mem / tMem t
                r = t { pList = new_plist, fMem = new_mem, fDsk = new_dsk,
                        failN1 = new_failn1, pMem = new_mp, pDsk = new_dp,
-                       uCpu = new_ucpu, pCpu = new_pcpu }
+                       uCpu = new_ucpu, pCpu = new_pcpu, utilLoad = new_load }
            in T.OpGood r
 
 -- | Adds a secondary instance.
@@ -313,13 +318,16 @@ addSec t inst pdx =
         new_prem = fromIntegral new_rmem / tMem t
         new_failn1 = old_mem <= new_rmem
         new_dp = fromIntegral new_dsk / tDsk t
+        old_load = utilLoad t
+        new_load = old_load { T.dskWeight = T.dskWeight old_load +
+                                            T.dskWeight (Instance.util inst) }
     in if new_dsk <= 0 || mDsk t > new_dp then T.OpFail T.FailDisk
        else if new_failn1 && not (failN1 t) then T.OpFail T.FailMem
        else let new_slist = iname:sList t
                 r = t { sList = new_slist, fDsk = new_dsk,
                         peers = new_peers, failN1 = new_failn1,
                         rMem = new_rmem, pDsk = new_dp,
-                        pRem = new_prem }
+                        pRem = new_prem, utilLoad = new_load }
            in T.OpGood r
 
 -- * Stats functions

hbal.hs

@@ -72,6 +72,7 @@ options =
     , oMaxCpu
     , oMinDisk
     , oDiskMoves
+    , oDynuFile
     , oShowVer
     , oShowHelp
     ]

hscan.hs

@@ -135,7 +135,7 @@ main = do
               printf "%-*s " nlen name
               hFlush stdout
               input_data <- Rapi.loadData name
-              let ldresult = input_data >>= Loader.mergeData
+              let ldresult = input_data >>= Loader.mergeData []
               (case ldresult of
                  Bad err -> printf "\nError: failed to load data. \
                                    \Details:\n%s\n" err