From 2bbf77cc91262de0166e7439a97a38472037de61 Mon Sep 17 00:00:00 2001
From: Iustin Pop <iustin@google.com>
Date: Wed, 8 Jul 2009 16:34:46 +0200
Subject: [PATCH] hspace: switch output to shell-script format
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This (big) patch changes the output of hspace from text-format
(separated by β: β) to a shell-snippet, in βkey=valueβ format.
This will allow sourcing the output or parsing it via awk/sed/etc.
---
Ganeti/HTools/Types.hs | 2 +-
hspace.1 | 169 ++++++++++++++++++++++++++++-------------
hspace.hs | 145 +++++++++++++++++++++--------------
3 files changed, 206 insertions(+), 110 deletions(-)
diff --git a/Ganeti/HTools/Types.hs b/Ganeti/HTools/Types.hs
index f7f55f593..cb384e852 100644
--- a/Ganeti/HTools/Types.hs
+++ b/Ganeti/HTools/Types.hs
@@ -65,7 +65,7 @@ data FailMode = FailMem -- ^ Failed due to not enough RAM
| FailDisk -- ^ Failed due to not enough disk
| FailCPU -- ^ Failed due to not enough CPU capacity
| FailN1 -- ^ Failed due to not passing N1 checks
- deriving (Eq, Show)
+ deriving (Eq, Enum, Bounded, Show)
-- | Either-like data-type customized for our failure modes
data OpResult a = OpFail FailMode -- ^ Failed operation
diff --git a/hspace.1 b/hspace.1
index e496423f6..db4235821 100644
--- a/hspace.1
+++ b/hspace.1
@@ -28,8 +28,121 @@ cluster, until the point where we don't have any N+1 possible
allocation. It uses the exact same allocation algorithm as the hail
iallocator plugin.
-With default options, the output of the program is designed to be
-parseable; when the -p option is passed, this is no longer true.
+The output of the program is designed to interpreted as a shell
+fragment (or parsed as a \fIkey=value\fR file). Options which extend
+the output (e.g. -p, -v) will output the additional information on
+stderr (such that the stdout is still parseable).
+
+The following keys are available in the output of the script (all
+prefixed with \fIHTS_\fR):
+.TP
+.I SPEC_MEM, SPEC_DSK, SPEC_CPU, SPEC_RQN
+These represent the specifications of the instance model used for
+allocation (the memory, disk, cpu, requested nodes).
+
+.TP
+.I CLUSTER_MEM, CLUSTER_DSK, CLUSTER_CPU, CLUSTER_NODES
+These represent the total memory, disk, CPU count and total nodes in
+the cluster.
+
+.TP
+.I INI_SCORE, FIN_SCORE
+These are the initial (current) and final cluster score (see the hbal
+man page for details about the scoring algorithm).
+
+.TP
+.I INI_INST_CNT, FIN_INST_CNT
+The initial and final instance count.
+
+.TP
+.I INI_MEM_FREE, FIN_MEM_FREE
+The initial and final total free memory in the cluster (but this
+doesn't necessarily mean available for use).
+
+.TP
+.I INI_MEM_AVAIL, FIN_MEM_AVAIL
+The initial and final total available memory for allocation in the
+cluster. If allocating redundant instances, new instances could
+increase the reserved memory so it doesn't necessarily mean the
+entirety of this memory can be used for new instance allocations.
+
+.TP
+.I INI_MEM_RESVD, FIN_MEM_RESVD
+The initial and final reserved memory (for redundancy/N+1 purposes).
+
+.TP
+.I INI_MEM_INST, FIN_MEM_INST
+The initial and final memory used for instances (actual runtime used
+RAM).
+
+.TP
+.I INI_MEM_OVERHEAD, FIN_MEM_OVERHEAD
+The initial and final memory overhead - memory used for the node
+itself and unacounted memory (e.g. due to hypervisor overhead).
+
+.TP
+.I INI_MEM_EFF, HTS_INI_MEM_EFF
+The initial and final memory efficiency, represented as instance
+memory divided by total memory.
+
+.TP
+.I INI_DSK_FREE, INI_DSK_AVAIL, INI_DSK_RESVD, INI_DSK_INST, INI_DSK_EFF
+Initial disk stats, similar to the memory ones.
+
+.TP
+.I FIN_DSK_FREE, FIN_DSK_AVAIL, FIN_DSK_RESVD, FIN_DSK_INST, FIN_DSK_EFF
+Final disk stats, similar to the memory ones.
+
+.TP
+.I INI_CPU_INST, FIN_CPU_INST
+Initial and final number of virtual CPUs used by instances.
+
+.TP
+.I INI_CPU_EFF, FIN_CPU_EFF
+The initial and final CPU efficiency, represented as the count of
+virtual instance CPUs divided by the total physical CPU count.
+
+.TP
+.I INI_MNODE_MEM_AVAIL, FIN_MNODE_MEM_AVAIL
+The initial and final maximum per-node available memory. This is not
+very useful as a metric but can give an impression of the status of
+the nodes; as an example, this value restricts the maximum instance
+size that can be still created on the cluster.
+
+.TP
+.I INI_MNODE_DSK_AVAIL, FIN_MNODE_DSK_AVAIL
+Like the above but for disk.
+
+.TP
+.I ALLOC_USAGE
+The current usage represented as initial number of instances divided
+per final number of instances.
+
+.TP
+.I ALLOC_COUNT
+The number of instances allocated (delta between FIN_INST_CNT and
+INI_INST_CNT).
+
+.TP
+.I ALLOC_FAIL*_CNT
+For the last attemp at allocations (which would have increased
+FIN_INST_CNT with one, if it had succeeded), this is the count of the
+failure reasons per failure type; currently defined are FAILMEM,
+FAILDISK and FAILCPU which represent errors due to not enough memory,
+disk and CPUs, and FAILN1 which represents a non N+1 compliant cluster
+on which we can't allocate instances at all.
+
+.TP
+.I ALLOC_FAIL_REASON
+The reason for most of the failures, being one of the above FAIL*
+strings.
+
+.TP
+.I OK
+A marker representing the successful end of the computation, and
+having value "1". If this key is not present in the output it means
+that the computation failed and any values present should not be
+relied upon.
.SH OPTIONS
The options that can be passed to the program are as follows:
@@ -162,7 +275,7 @@ considered a fully-specified URL and is used as-is.
Increase the output verbosity. Each usage of this option will increase
the verbosity (currently more than 2 doesn't make sense) from the
default of one. At verbosity 2 the location of the new instances is
-shown in program output.
+shown in the standard error.
.TP
.B -q, --quiet
@@ -189,56 +302,6 @@ The algorithm doesn't rebalance the cluster or try to get the optimal
fit; it just allocates in the best place for the current step, without
taking into consideration the impact on future placements.
-.SH EXAMPLE
-
-.SS Default output
-
-.in +4n
-.nf
-.RB "$" " hspace --mem 16 --disk 16 --req-nodes 2"
-Initial score: 0.38988095
-Initial instances: 3
-Initial free RAM: 546
-Initial free disk: 260600
-Final score: 0.32638889
-Final instances: 7
-Final free RAM: 482
-Final free disk: 260472
-Usage: 0.43
-Allocations: 4
-.fi
-.in
-
-This shows that (on this fake cluster), starting from 3 initial
-instances, using the hail iallocator plugin, it would be possible to
-add four (Allocations: 4) new instances to the cluster.
-
-.SS Verbose output
-
-For the same cluster as above:
-.in +4n
-.nf
-.RB "$" " hspace --mem 16 --disk 16 --req-nodes 2 -v"
-Initial score: 0.38988095
-Initial instances: 3
-Initial free RAM: 546
-Initial free disk: 260600
-Final score: 0.32638889
-Final instances: 7
-Final free RAM: 482
-Final free disk: 260472
-Usage: 0.43
-Allocations: 4
-Inst: new-0 node2 node1
-Inst: new-1 node2 node1
-Inst: new-2 node2 node1
-Inst: new-3 node2 node1
-.fi
-.in
-
-The output now includes the placement for the new instances (named
-\fBnew-\fInumber\fR).
-
.SH ENVIRONMENT
If the variables \fBHTOOLS_NODES\fR and \fBHTOOLS_INSTANCES\fR are
diff --git a/hspace.hs b/hspace.hs
index 4b1417894..8159f4418 100644
--- a/hspace.hs
+++ b/hspace.hs
@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
module Main (main) where
+import Data.Char (toUpper)
import Data.List
import Data.Function
import Monad
@@ -147,6 +148,50 @@ options =
"show help"
]
+data Phase = PInitial | PFinal
+
+statsData :: [(String, Cluster.CStats -> String)]
+statsData = [ ("SCORE", printf "%.8f" . Cluster.cs_score)
+ , ("INST_CNT", printf "%d" . Cluster.cs_ninst)
+ , ("MEM_FREE", printf "%d" . Cluster.cs_fmem)
+ , ("MEM_AVAIL", printf "%d" . Cluster.cs_amem)
+ , ("MEM_RESVD",
+ \cs -> printf "%d" (Cluster.cs_fmem cs - Cluster.cs_amem cs))
+ , ("MEM_INST", printf "%d" . Cluster.cs_imem)
+ , ("MEM_OVERHEAD",
+ \cs -> printf "%d" (Cluster.cs_xmem cs + Cluster.cs_nmem cs))
+ , ("MEM_EFF",
+ \cs -> printf "%.8f" (fromIntegral (Cluster.cs_imem cs) /
+ Cluster.cs_tmem cs))
+ , ("DSK_FREE", printf "%d" . Cluster.cs_fdsk)
+ , ("DSK_AVAIL", printf "%d ". Cluster.cs_adsk)
+ , ("DSK_RESVD",
+ \cs -> printf "%d" (Cluster.cs_fdsk cs - Cluster.cs_adsk cs))
+ , ("DSK_INST", printf "%d" . Cluster.cs_idsk)
+ , ("DSK_EFF",
+ \cs -> printf "%.8f" (fromIntegral (Cluster.cs_idsk cs) /
+ Cluster.cs_tdsk cs))
+ , ("CPU_INST", printf "%d" . Cluster.cs_icpu)
+ , ("CPU_EFF",
+ \cs -> printf "%.8f" (fromIntegral (Cluster.cs_icpu cs) /
+ Cluster.cs_tcpu cs))
+ , ("MNODE_MEM_AVAIL", printf "%d" . Cluster.cs_mmem)
+ , ("MNODE_DSK_AVAIL", printf "%d" . Cluster.cs_mdsk)
+ ]
+
+specData :: [(String, Options -> String)]
+specData = [ ("MEM", printf "%d" . optIMem)
+ , ("DSK", printf "%d" . optIDsk)
+ , ("CPU", printf "%d" . optIVCPUs)
+ , ("RQN", printf "%d" . optINodes)
+ ]
+
+clusterData :: [(String, Cluster.CStats -> String)]
+clusterData = [ ("MEM", printf "%.0f" . Cluster.cs_tmem)
+ , ("DSK", printf "%.0f" . Cluster.cs_tdsk)
+ , ("CPU", printf "%.0f" . Cluster.cs_tcpu)
+ ]
+
-- | Build failure stats out of a list of failure reasons
concatFailure :: [(FailMode, Int)] -> FailMode -> [(FailMode, Int)]
concatFailure flst reason =
@@ -169,7 +214,7 @@ filterFails sols =
) $ sols
aval = concat alst
bval = concat blst
- in (foldl' concatFailure [] aval, bval)
+ in (foldl' concatFailure [(x, 0) | x <- [minBound..maxBound]] aval, bval)
-- | Get the placement with best score out of a list of possible placements
processResults :: [(Node.List, Instance.Instance, [Node.Node])]
@@ -203,30 +248,12 @@ iterateDepth nl il newinst nreq ixes =
in iterateDepth xnl il newinst nreq (xi:ixes)
-- | Function to print stats for a given phase
-printStats :: String -> Cluster.CStats -> IO ()
-printStats kind cs = do
- printf "%s score: %.8f\n" kind (Cluster.cs_score cs)
- printf "%s instances: %d\n" kind (Cluster.cs_ninst cs)
- printf "%s free RAM: %d\n" kind (Cluster.cs_fmem cs)
- printf "%s allocatable RAM: %d\n" kind (Cluster.cs_amem cs)
- printf "%s reserved RAM: %d\n" kind (Cluster.cs_fmem cs -
- Cluster.cs_amem cs)
- printf "%s instance RAM: %d\n" kind (Cluster.cs_imem cs)
- printf "%s overhead RAM: %d\n" kind (Cluster.cs_xmem cs + Cluster.cs_nmem cs)
- printf "%s RAM usage efficiency: %.8f\n"
- kind (fromIntegral (Cluster.cs_imem cs) / Cluster.cs_tmem cs)
- printf "%s free disk: %d\n" kind (Cluster.cs_fdsk cs)
- printf "%s allocatable disk: %d\n" kind (Cluster.cs_adsk cs)
- printf "%s reserved disk: %d\n" kind (Cluster.cs_fdsk cs -
- Cluster.cs_adsk cs)
- printf "%s instance disk: %d\n" kind (Cluster.cs_idsk cs)
- printf "%s disk usage efficiency: %.8f\n"
- kind (fromIntegral (Cluster.cs_idsk cs) / Cluster.cs_tdsk cs)
- printf "%s instance cpus: %d\n" kind (Cluster.cs_icpu cs)
- printf "%s cpu usage efficiency: %.8f\n"
- kind (fromIntegral (Cluster.cs_icpu cs) / Cluster.cs_tcpu cs)
- printf "%s max node allocatable RAM: %d\n" kind (Cluster.cs_mmem cs)
- printf "%s max node allocatable disk: %d\n" kind (Cluster.cs_mdsk cs)
+printStats :: Phase -> Cluster.CStats -> [(String, String)]
+printStats ph cs =
+ map (\(s, fn) -> (printf "%s_%s" kind s, fn cs)) statsData
+ where kind = case ph of
+ PInitial -> "INI"
+ PFinal -> "FIN"
-- | Print final stats and related metrics
printResults :: Node.List -> Int -> Int -> [(FailMode, Int)] -> IO ()
@@ -241,12 +268,21 @@ printResults fin_nl num_instances allocs sreason = do
(Cluster.cs_ninst fin_stats)
exitWith $ ExitFailure 1
- printStats "Final" fin_stats
- printf "Usage: %.5f\n" ((fromIntegral num_instances::Double) /
- fromIntegral fin_instances)
- printf "Allocations: %d\n" allocs
- putStr (unlines . map (\(x, y) -> printf "%s: %d" (show x) y) $ sreason)
- printf "Most likely fail reason: %s\n" (show . fst . head $ sreason)
+ printKeys $ printStats PFinal fin_stats
+ printKeys [ ("ALLOC_USAGE", printf "%.8f"
+ ((fromIntegral num_instances::Double) /
+ fromIntegral fin_instances))
+ , ("ALLOC_COUNT", printf "%d" allocs)
+ , ("ALLOC_FAIL_REASON", map toUpper . show . fst $ head sreason)
+ ]
+ printKeys $ map (\(x, y) -> (printf "ALLOC_%s_CNT" (show x),
+ printf "%d" y)) sreason
+ -- this should be the final entry
+ printKeys [("OK", "1")]
+
+-- | Format a list of key/values as a shell fragment
+printKeys :: [(String, String)] -> IO ()
+printKeys = mapM_ (\(k, v) -> printf "HTS_%s=%s\n" (map toUpper k) v)
-- | Main function.
main :: IO ()
@@ -262,10 +298,7 @@ main = do
(fixed_nl, il, csf) <- CLI.loadExternalData opts
- printf "Spec RAM: %d\n" (optIMem opts)
- printf "Spec disk: %d\n" (optIDsk opts)
- printf "Spec CPUs: %d\n" (optIVCPUs opts)
- printf "Spec nodes: %d\n" (optINodes opts)
+ printKeys $ map (\(a, fn) -> ("SPEC_" ++ a, fn opts)) specData
let num_instances = length $ Container.elems il
@@ -296,24 +329,23 @@ main = do
nm
when (length csf > 0 && verbose > 1) $
- printf "Note: Stripping common suffix of '%s' from names\n" csf
+ hPrintf stderr "Note: Stripping common suffix of '%s' from names\n" csf
when (optShowNodes opts) $
do
- putStrLn "Initial cluster status:"
- putStrLn $ Cluster.printNodes nl
+ hPutStrLn stderr "Initial cluster status:"
+ hPutStrLn stderr $ Cluster.printNodes nl
let ini_cv = Cluster.compCV nl
ini_stats = Cluster.totalResources nl
when (verbose > 2) $ do
- printf "Initial coefficients: overall %.8f, %s\n"
- ini_cv (Cluster.printStats nl)
+ hPrintf stderr "Initial coefficients: overall %.8f, %s\n"
+ ini_cv (Cluster.printStats nl)
- printf "Cluster RAM: %.0f\n" (Cluster.cs_tmem ini_stats)
- printf "Cluster disk: %.0f\n" (Cluster.cs_tdsk ini_stats)
- printf "Cluster cpus: %.0f\n" (Cluster.cs_tcpu ini_stats)
- printStats "Initial" ini_stats
+ printKeys $ map (\(a, fn) -> ("CLUSTER_" ++ a, fn ini_stats)) clusterData
+ printKeys [("CLUSTER_NODES", printf "%d" (length all_nodes))]
+ printKeys $ printStats PInitial ini_stats
let bad_nodes = fst $ Cluster.computeBadItems nl il
when (length bad_nodes > 0) $ do
@@ -332,19 +364,20 @@ main = do
ix_namelen = maximum . map (length . Instance.name) $ fin_ixes
sreason = reverse $ sortBy (compare `on` snd) ereason
- printResults fin_nl num_instances allocs sreason
-
when (verbose > 1) $
- putStr . unlines . map (\i -> printf "Inst: %*s %-*s %-*s"
- ix_namelen (Instance.name i)
- nmlen (Container.nameOf fin_nl $ Instance.pnode i)
- nmlen (let sdx = Instance.snode i
- in if sdx == Node.noSecondary then ""
- else Container.nameOf fin_nl sdx))
- $ fin_ixes
+ hPutStr stderr . unlines $
+ map (\i -> printf "Inst: %*s %-*s %-*s"
+ ix_namelen (Instance.name i)
+ nmlen (Container.nameOf fin_nl $ Instance.pnode i)
+ nmlen (let sdx = Instance.snode i
+ in if sdx == Node.noSecondary then ""
+ else Container.nameOf fin_nl sdx)
+ ) fin_ixes
when (optShowNodes opts) $
do
- putStrLn ""
- putStrLn "Final cluster status:"
- putStrLn $ Cluster.printNodes fin_nl
+ hPutStrLn stderr ""
+ hPutStrLn stderr "Final cluster status:"
+ hPutStrLn stderr $ Cluster.printNodes fin_nl
+
+ printResults fin_nl num_instances allocs sreason
--
GitLab