QC.hs 34.4 KB
Newer Older
1
{-| Unittests for ganeti-htools.
Iustin Pop's avatar
Iustin Pop committed
2
3
4
5
6

-}

{-

7
Copyright (C) 2009, 2010, 2011 Google Inc.
Iustin Pop's avatar
Iustin Pop committed
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.

-}

26
module Ganeti.HTools.QC
27
28
    ( testUtils
    , testPeerMap
29
30
31
32
    , testContainer
    , testInstance
    , testNode
    , testText
Iustin Pop's avatar
Iustin Pop committed
33
    , testOpCodes
34
    , testJobs
35
    , testCluster
Iustin Pop's avatar
Iustin Pop committed
36
    , testLoader
37
    , testTypes
38
    ) where
39
40

import Test.QuickCheck
Iustin Pop's avatar
Iustin Pop committed
41
import Data.List (findIndex, intercalate, nub, isPrefixOf)
42
import Data.Maybe
Iustin Pop's avatar
Iustin Pop committed
43
44
import Control.Monad
import qualified Text.JSON as J
Iustin Pop's avatar
Iustin Pop committed
45
import qualified Data.Map
46
import qualified Data.IntMap as IntMap
Iustin Pop's avatar
Iustin Pop committed
47
import qualified Ganeti.OpCodes as OpCodes
48
import qualified Ganeti.Jobs as Jobs
Iustin Pop's avatar
Iustin Pop committed
49
import qualified Ganeti.Luxi
50
51
52
import qualified Ganeti.HTools.CLI as CLI
import qualified Ganeti.HTools.Cluster as Cluster
import qualified Ganeti.HTools.Container as Container
Iustin Pop's avatar
Iustin Pop committed
53
import qualified Ganeti.HTools.ExtLoader
54
55
56
import qualified Ganeti.HTools.IAlloc as IAlloc
import qualified Ganeti.HTools.Instance as Instance
import qualified Ganeti.HTools.Loader as Loader
Iustin Pop's avatar
Iustin Pop committed
57
import qualified Ganeti.HTools.Luxi
58
import qualified Ganeti.HTools.Node as Node
Iustin Pop's avatar
Iustin Pop committed
59
import qualified Ganeti.HTools.Group as Group
60
import qualified Ganeti.HTools.PeerMap as PeerMap
61
import qualified Ganeti.HTools.Rapi
Iustin Pop's avatar
Iustin Pop committed
62
import qualified Ganeti.HTools.Simu
63
64
65
import qualified Ganeti.HTools.Text as Text
import qualified Ganeti.HTools.Types as Types
import qualified Ganeti.HTools.Utils as Utils
Iustin Pop's avatar
Iustin Pop committed
66
import qualified Ganeti.HTools.Version
67
import qualified Ganeti.Constants as C
68

Iustin Pop's avatar
Iustin Pop committed
69
70
71
run :: Testable prop => prop -> Args -> IO Result
run = flip quickCheckWithResult

72
73
-- * Constants

74
-- | Maximum memory (1TiB, somewhat random value).
Iustin Pop's avatar
Iustin Pop committed
75
76
77
maxMem :: Int
maxMem = 1024 * 1024

78
-- | Maximum disk (8TiB, somewhat random value).
Iustin Pop's avatar
Iustin Pop committed
79
maxDsk :: Int
Iustin Pop's avatar
Iustin Pop committed
80
maxDsk = 1024 * 1024 * 8
Iustin Pop's avatar
Iustin Pop committed
81

82
-- | Max CPUs (1024, somewhat random value).
Iustin Pop's avatar
Iustin Pop committed
83
84
85
maxCpu :: Int
maxCpu = 1024

Iustin Pop's avatar
Iustin Pop committed
86
87
88
89
90
91
defGroup :: Group.Group
defGroup = flip Group.setIdx 0 $
               Group.create "default" Utils.defaultGroupID
                    Types.AllocPreferred

defGroupList :: Group.List
Iustin Pop's avatar
Iustin Pop committed
92
defGroupList = Container.fromList [(Group.idx defGroup, defGroup)]
Iustin Pop's avatar
Iustin Pop committed
93
94
95
96

defGroupAssoc :: Data.Map.Map String Types.Gdx
defGroupAssoc = Data.Map.singleton (Group.uuid defGroup) (Group.idx defGroup)

97
98
-- * Helper functions

99
-- | Simple checker for whether OpResult is fail or pass.
Iustin Pop's avatar
Iustin Pop committed
100
101
102
103
isFailure :: Types.OpResult a -> Bool
isFailure (Types.OpFail _) = True
isFailure _ = False

104
-- | Update an instance to be smaller than a node.
105
setInstanceSmallerThanNode node inst =
106
107
108
    inst { Instance.mem = Node.availMem node `div` 2
         , Instance.dsk = Node.availDisk node `div` 2
         , Instance.vcpus = Node.availCpu node `div` 2
109
110
         }

111
-- | Create an instance given its spec.
112
createInstance mem dsk vcpus =
113
    Instance.create "inst-unnamed" mem dsk vcpus "running" [] True (-1) (-1)
114
                    Types.DTDrbd8
115

116
-- | Create a small cluster by repeating a node spec.
117
118
119
120
121
makeSmallCluster :: Node.Node -> Int -> Node.List
makeSmallCluster node count =
    let fn = Node.buildPeers node Container.empty
        namelst = map (\n -> (Node.name n, n)) (replicate count fn)
        (_, nlst) = Loader.assignIndices namelst
122
    in nlst
123

124
-- | Checks if a node is "big" enough.
125
126
127
128
129
isNodeBig :: Node.Node -> Int -> Bool
isNodeBig node size = Node.availDisk node > size * Types.unitDsk
                      && Node.availMem node > size * Types.unitMem
                      && Node.availCpu node > size * Types.unitCpu

130
131
canBalance :: Cluster.Table -> Bool -> Bool -> Bool -> Bool
canBalance tbl dm im evac = isJust $ Cluster.tryBalance tbl dm im evac 0 0
132

133
-- | Assigns a new fresh instance to a cluster; this is not
134
-- allocation, so no resource checks are done.
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
assignInstance :: Node.List -> Instance.List -> Instance.Instance ->
                  Types.Idx -> Types.Idx ->
                  (Node.List, Instance.List)
assignInstance nl il inst pdx sdx =
  let pnode = Container.find pdx nl
      snode = Container.find sdx nl
      maxiidx = if Container.null il
                then 0
                else fst (Container.findMax il) + 1
      inst' = inst { Instance.idx = maxiidx,
                     Instance.pNode = pdx, Instance.sNode = sdx }
      pnode' = Node.setPri pnode inst'
      snode' = Node.setSec snode inst'
      nl' = Container.addTwo pdx pnode' sdx snode' nl
      il' = Container.add maxiidx inst' il
  in (nl', il')

152
153
-- * Arbitrary instances

154
-- | Defines a DNS name.
155
newtype DNSChar = DNSChar { dnsGetChar::Char }
156

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
instance Arbitrary DNSChar where
    arbitrary = do
      x <- elements (['a'..'z'] ++ ['0'..'9'] ++ "_-")
      return (DNSChar x)

getName :: Gen String
getName = do
  n <- choose (1, 64)
  dn <- vector n::Gen [DNSChar]
  return (map dnsGetChar dn)


getFQDN :: Gen String
getFQDN = do
  felem <- getName
  ncomps <- choose (1, 4)
  frest <- vector ncomps::Gen [[DNSChar]]
  let frest' = map (map dnsGetChar) frest
  return (felem ++ "." ++ intercalate "." frest')

177
178
179
-- let's generate a random instance
instance Arbitrary Instance.Instance where
    arbitrary = do
180
      name <- getFQDN
Iustin Pop's avatar
Iustin Pop committed
181
182
      mem <- choose (0, maxMem)
      dsk <- choose (0, maxDsk)
183
184
185
186
187
188
189
190
      run_st <- elements [ C.inststErrorup
                         , C.inststErrordown
                         , C.inststAdmindown
                         , C.inststNodedown
                         , C.inststNodeoffline
                         , C.inststRunning
                         , "no_such_status1"
                         , "no_such_status2"]
191
192
      pn <- arbitrary
      sn <- arbitrary
Iustin Pop's avatar
Iustin Pop committed
193
      vcpus <- choose (0, maxCpu)
194
      return $ Instance.create name mem dsk vcpus run_st [] True pn sn
195
                               Types.DTDrbd8
196

197
198
199
200
201
-- | Generas an arbitrary node based on sizing information.
genNode :: Maybe Int -- ^ Minimum node size in terms of units
        -> Maybe Int -- ^ Maximum node size (when Nothing, bounded
                     -- just by the max... constants)
        -> Gen Node.Node
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
genNode min_multiplier max_multiplier = do
  let (base_mem, base_dsk, base_cpu) =
          case min_multiplier of
            Just mm -> (mm * Types.unitMem,
                        mm * Types.unitDsk,
                        mm * Types.unitCpu)
            Nothing -> (0, 0, 0)
      (top_mem, top_dsk, top_cpu)  =
          case max_multiplier of
            Just mm -> (mm * Types.unitMem,
                        mm * Types.unitDsk,
                        mm * Types.unitCpu)
            Nothing -> (maxMem, maxDsk, maxCpu)
  name  <- getFQDN
  mem_t <- choose (base_mem, top_mem)
  mem_f <- choose (base_mem, mem_t)
  mem_n <- choose (0, mem_t - mem_f)
  dsk_t <- choose (base_dsk, top_dsk)
  dsk_f <- choose (base_dsk, dsk_t)
  cpu_t <- choose (base_cpu, top_cpu)
  offl  <- arbitrary
  let n = Node.create name (fromIntegral mem_t) mem_n mem_f
          (fromIntegral dsk_t) dsk_f (fromIntegral cpu_t) offl 0
  return $ Node.buildPeers n Container.empty

227
228
-- and a random node
instance Arbitrary Node.Node where
229
    arbitrary = genNode Nothing Nothing
230

Iustin Pop's avatar
Iustin Pop committed
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
-- replace disks
instance Arbitrary OpCodes.ReplaceDisksMode where
  arbitrary = elements [ OpCodes.ReplaceOnPrimary
                       , OpCodes.ReplaceOnSecondary
                       , OpCodes.ReplaceNewSecondary
                       , OpCodes.ReplaceAuto
                       ]

instance Arbitrary OpCodes.OpCode where
  arbitrary = do
    op_id <- elements [ "OP_TEST_DELAY"
                      , "OP_INSTANCE_REPLACE_DISKS"
                      , "OP_INSTANCE_FAILOVER"
                      , "OP_INSTANCE_MIGRATE"
                      ]
    (case op_id of
        "OP_TEST_DELAY" ->
          liftM3 OpCodes.OpTestDelay arbitrary arbitrary arbitrary
        "OP_INSTANCE_REPLACE_DISKS" ->
250
          liftM5 OpCodes.OpInstanceReplaceDisks arbitrary arbitrary
Iustin Pop's avatar
Iustin Pop committed
251
252
          arbitrary arbitrary arbitrary
        "OP_INSTANCE_FAILOVER" ->
253
          liftM2 OpCodes.OpInstanceFailover arbitrary arbitrary
Iustin Pop's avatar
Iustin Pop committed
254
        "OP_INSTANCE_MIGRATE" ->
255
          liftM4 OpCodes.OpInstanceMigrate arbitrary arbitrary arbitrary
256
          arbitrary
Iustin Pop's avatar
Iustin Pop committed
257
258
        _ -> fail "Wrong opcode")

259
260
261
262
263
264
instance Arbitrary Jobs.OpStatus where
  arbitrary = elements [minBound..maxBound]

instance Arbitrary Jobs.JobStatus where
  arbitrary = elements [minBound..maxBound]

265
266
267
268
269
270
newtype SmallRatio = SmallRatio Double deriving Show
instance Arbitrary SmallRatio where
    arbitrary = do
      v <- choose (0, 1)
      return $ SmallRatio v

271
272
273
274
275
276
instance Arbitrary Types.AllocPolicy where
  arbitrary = elements [minBound..maxBound]

instance Arbitrary Types.DiskTemplate where
  arbitrary = elements [minBound..maxBound]

277
-- * Actual tests
Iustin Pop's avatar
Iustin Pop committed
278

279
280
281
282
-- ** Utils tests

-- | If the list is not just an empty element, and if the elements do
-- not contain commas, then join+split should be idempotent.
Iustin Pop's avatar
Iustin Pop committed
283
284
285
286
287
prop_Utils_commaJoinSplit =
    forAll (arbitrary `suchThat`
            (\l -> l /= [""] && all (not . elem ',') l )) $ \lst ->
    Utils.sepSplit ',' (Utils.commaJoin lst) == lst

288
-- | Split and join should always be idempotent.
289
290
prop_Utils_commaSplitJoin s = Utils.commaJoin (Utils.sepSplit ',' s) == s

291
-- | fromObjWithDefault, we test using the Maybe monad and an integer
292
-- value.
293
294
295
296
297
298
prop_Utils_fromObjWithDefault def_value random_key =
    -- a missing key will be returned with the default
    Utils.fromObjWithDefault [] random_key def_value == Just def_value &&
    -- a found key will be returned as is, not with default
    Utils.fromObjWithDefault [(random_key, J.showJSON def_value)]
         random_key (def_value+1) == Just def_value
Iustin Pop's avatar
Iustin Pop committed
299
        where _types = def_value :: Integer
300

301
-- | Test list for the Utils module.
302
303
304
testUtils =
  [ run prop_Utils_commaJoinSplit
  , run prop_Utils_commaSplitJoin
305
  , run prop_Utils_fromObjWithDefault
306
307
  ]

308
309
310
-- ** PeerMap tests

-- | Make sure add is idempotent.
Iustin Pop's avatar
Iustin Pop committed
311
prop_PeerMap_addIdempotent pmap key em =
312
    fn puniq == fn (fn puniq)
313
    where _types = (pmap::PeerMap.PeerMap,
Iustin Pop's avatar
Iustin Pop committed
314
315
                    key::PeerMap.Key, em::PeerMap.Elem)
          fn = PeerMap.add key em
316
          puniq = PeerMap.accumArray const pmap
317

318
-- | Make sure remove is idempotent.
319
320
prop_PeerMap_removeIdempotent pmap key =
    fn puniq == fn (fn puniq)
321
322
    where _types = (pmap::PeerMap.PeerMap, key::PeerMap.Key)
          fn = PeerMap.remove key
323
324
          puniq = PeerMap.accumArray const pmap

325
-- | Make sure a missing item returns 0.
326
327
prop_PeerMap_findMissing pmap key =
    PeerMap.find key (PeerMap.remove key puniq) == 0
328
    where _types = (pmap::PeerMap.PeerMap, key::PeerMap.Key)
329
330
          puniq = PeerMap.accumArray const pmap

331
-- | Make sure an added item is found.
Iustin Pop's avatar
Iustin Pop committed
332
333
prop_PeerMap_addFind pmap key em =
    PeerMap.find key (PeerMap.add key em puniq) == em
334
    where _types = (pmap::PeerMap.PeerMap,
Iustin Pop's avatar
Iustin Pop committed
335
                    key::PeerMap.Key, em::PeerMap.Elem)
336
          puniq = PeerMap.accumArray const pmap
337

338
-- | Manual check that maxElem returns the maximum indeed, or 0 for null.
339
340
341
prop_PeerMap_maxElem pmap =
    PeerMap.maxElem puniq == if null puniq then 0
                             else (maximum . snd . unzip) puniq
342
    where _types = pmap::PeerMap.PeerMap
343
344
          puniq = PeerMap.accumArray const pmap

345
-- | List of tests for the PeerMap module.
346
testPeerMap =
347
348
349
350
351
352
353
    [ run prop_PeerMap_addIdempotent
    , run prop_PeerMap_removeIdempotent
    , run prop_PeerMap_maxElem
    , run prop_PeerMap_addFind
    , run prop_PeerMap_findMissing
    ]

354
-- ** Container tests
355
356
357
358
359
360
361
362
363

prop_Container_addTwo cdata i1 i2 =
    fn i1 i2 cont == fn i2 i1 cont &&
       fn i1 i2 cont == fn i1 i2 (fn i1 i2 cont)
    where _types = (cdata::[Int],
                    i1::Int, i2::Int)
          cont = foldl (\c x -> Container.add x x c) Container.empty cdata
          fn x1 x2 = Container.addTwo x1 x1 x2 x2

364
365
366
367
368
prop_Container_nameOf node =
  let nl = makeSmallCluster node 1
      fnode = head (Container.elems nl)
  in Container.nameOf nl (Node.idx fnode) == Node.name fnode

369
-- | We test that in a cluster, given a random node, we can find it by
370
-- its name and alias, as long as all names and aliases are unique,
371
-- and that we fail to find a non-existing name.
372
373
374
375
376
377
378
379
380
381
382
383
384
prop_Container_findByName node othername =
  forAll (choose (1, 20)) $ \ cnt ->
  forAll (choose (0, cnt - 1)) $ \ fidx ->
  forAll (vector cnt) $ \ names ->
  (length . nub) (map fst names ++ map snd names) ==
  length names * 2 &&
  not (othername `elem` (map fst names ++ map snd names)) ==>
  let nl = makeSmallCluster node cnt
      nodes = Container.elems nl
      nodes' = map (\((name, alias), nn) -> (Node.idx nn,
                                             nn { Node.name = name,
                                                  Node.alias = alias }))
               $ zip names nodes
Iustin Pop's avatar
Iustin Pop committed
385
      nl' = Container.fromList nodes'
386
387
388
389
390
      target = snd (nodes' !! fidx)
  in Container.findByName nl' (Node.name target) == Just target &&
     Container.findByName nl' (Node.alias target) == Just target &&
     Container.findByName nl' othername == Nothing

391
testContainer =
392
393
394
395
    [ run prop_Container_addTwo
    , run prop_Container_nameOf
    , run prop_Container_findByName
    ]
396

397
398
-- ** Instance tests

399
400
-- Simple instance tests, we only have setter/getters

Iustin Pop's avatar
Iustin Pop committed
401
402
403
prop_Instance_creat inst =
    Instance.name inst == Instance.alias inst

404
405
406
407
408
prop_Instance_setIdx inst idx =
    Instance.idx (Instance.setIdx inst idx) == idx
    where _types = (inst::Instance.Instance, idx::Types.Idx)

prop_Instance_setName inst name =
Iustin Pop's avatar
Iustin Pop committed
409
410
411
412
413
414
415
416
    Instance.name newinst == name &&
    Instance.alias newinst == name
    where _types = (inst::Instance.Instance, name::String)
          newinst = Instance.setName inst name

prop_Instance_setAlias inst name =
    Instance.name newinst == Instance.name inst &&
    Instance.alias newinst == name
417
    where _types = (inst::Instance.Instance, name::String)
Iustin Pop's avatar
Iustin Pop committed
418
          newinst = Instance.setAlias inst name
419
420

prop_Instance_setPri inst pdx =
421
    Instance.pNode (Instance.setPri inst pdx) == pdx
422
423
424
    where _types = (inst::Instance.Instance, pdx::Types.Ndx)

prop_Instance_setSec inst sdx =
425
    Instance.sNode (Instance.setSec inst sdx) == sdx
426
427
428
    where _types = (inst::Instance.Instance, sdx::Types.Ndx)

prop_Instance_setBoth inst pdx sdx =
429
    Instance.pNode si == pdx && Instance.sNode si == sdx
430
431
432
    where _types = (inst::Instance.Instance, pdx::Types.Ndx, sdx::Types.Ndx)
          si = Instance.setBoth inst pdx sdx

Iustin Pop's avatar
Iustin Pop committed
433
434
435
436
prop_Instance_runStatus_True =
    forAll (arbitrary `suchThat`
            ((`elem` Instance.runningStates) . Instance.runSt))
    Instance.running
Iustin Pop's avatar
Iustin Pop committed
437
438
439

prop_Instance_runStatus_False inst =
    let run_st = Instance.running inst
440
        run_tx = Instance.runSt inst
Iustin Pop's avatar
Iustin Pop committed
441
    in
442
      run_tx `notElem` Instance.runningStates ==> not run_st
Iustin Pop's avatar
Iustin Pop committed
443

Iustin Pop's avatar
Iustin Pop committed
444
445
446
447
448
449
450
451
prop_Instance_shrinkMG inst =
    Instance.mem inst >= 2 * Types.unitMem ==>
        case Instance.shrinkByType inst Types.FailMem of
          Types.Ok inst' ->
              Instance.mem inst' == Instance.mem inst - Types.unitMem
          _ -> False

prop_Instance_shrinkMF inst =
Iustin Pop's avatar
Iustin Pop committed
452
453
454
    forAll (choose (0, 2 * Types.unitMem - 1)) $ \mem ->
    let inst' = inst { Instance.mem = mem}
    in Types.isBad $ Instance.shrinkByType inst' Types.FailMem
Iustin Pop's avatar
Iustin Pop committed
455
456
457
458
459
460
461
462
463

prop_Instance_shrinkCG inst =
    Instance.vcpus inst >= 2 * Types.unitCpu ==>
        case Instance.shrinkByType inst Types.FailCPU of
          Types.Ok inst' ->
              Instance.vcpus inst' == Instance.vcpus inst - Types.unitCpu
          _ -> False

prop_Instance_shrinkCF inst =
Iustin Pop's avatar
Iustin Pop committed
464
465
466
    forAll (choose (0, 2 * Types.unitCpu - 1)) $ \vcpus ->
    let inst' = inst { Instance.vcpus = vcpus }
    in Types.isBad $ Instance.shrinkByType inst' Types.FailCPU
Iustin Pop's avatar
Iustin Pop committed
467
468
469
470
471
472
473
474
475

prop_Instance_shrinkDG inst =
    Instance.dsk inst >= 2 * Types.unitDsk ==>
        case Instance.shrinkByType inst Types.FailDisk of
          Types.Ok inst' ->
              Instance.dsk inst' == Instance.dsk inst - Types.unitDsk
          _ -> False

prop_Instance_shrinkDF inst =
Iustin Pop's avatar
Iustin Pop committed
476
477
478
    forAll (choose (0, 2 * Types.unitDsk - 1)) $ \dsk ->
    let inst' = inst { Instance.dsk = dsk }
    in Types.isBad $ Instance.shrinkByType inst' Types.FailDisk
Iustin Pop's avatar
Iustin Pop committed
479
480
481

prop_Instance_setMovable inst m =
    Instance.movable inst' == m
482
    where inst' = Instance.setMovable inst m
Iustin Pop's avatar
Iustin Pop committed
483

484
testInstance =
Iustin Pop's avatar
Iustin Pop committed
485
486
    [ run prop_Instance_creat
    , run prop_Instance_setIdx
487
    , run prop_Instance_setName
Iustin Pop's avatar
Iustin Pop committed
488
    , run prop_Instance_setAlias
489
490
491
    , run prop_Instance_setPri
    , run prop_Instance_setSec
    , run prop_Instance_setBoth
Iustin Pop's avatar
Iustin Pop committed
492
493
    , run prop_Instance_runStatus_True
    , run prop_Instance_runStatus_False
Iustin Pop's avatar
Iustin Pop committed
494
495
496
497
498
499
500
    , run prop_Instance_shrinkMG
    , run prop_Instance_shrinkMF
    , run prop_Instance_shrinkCG
    , run prop_Instance_shrinkCF
    , run prop_Instance_shrinkDG
    , run prop_Instance_shrinkDF
    , run prop_Instance_setMovable
Iustin Pop's avatar
Iustin Pop committed
501
502
    ]

503
504
-- ** Text backend tests

Iustin Pop's avatar
Iustin Pop committed
505
506
-- Instance text loader tests

Iustin Pop's avatar
Iustin Pop committed
507
508
509
prop_Text_Load_Instance name mem dsk vcpus status
                        (NonEmpty pnode) snode
                        (NonNegative pdx) (NonNegative sdx) autobal =
510
    pnode /= snode && pdx /= sdx ==>
Iustin Pop's avatar
Iustin Pop committed
511
512
513
    let vcpus_s = show vcpus
        dsk_s = show dsk
        mem_s = show mem
Iustin Pop's avatar
Iustin Pop committed
514
515
        ndx = if null snode
              then [(pnode, pdx)]
516
              else [(pnode, pdx), (snode, sdx)]
517
        nl = Data.Map.fromList ndx
518
        tags = ""
Iustin Pop's avatar
Iustin Pop committed
519
        sbal = if autobal then "Y" else "N"
520
        inst = Text.loadInst nl
Iustin Pop's avatar
Iustin Pop committed
521
522
               [name, mem_s, dsk_s, vcpus_s, status,
                sbal, pnode, snode, tags]:: Maybe (String, Instance.Instance)
523
        fail1 = Text.loadInst nl
Iustin Pop's avatar
Iustin Pop committed
524
525
               [name, mem_s, dsk_s, vcpus_s, status,
                sbal, pnode, pnode, tags]:: Maybe (String, Instance.Instance)
Iustin Pop's avatar
Iustin Pop committed
526
527
        _types = ( name::String, mem::Int, dsk::Int
                 , vcpus::Int, status::String
Iustin Pop's avatar
Iustin Pop committed
528
                 , snode::String
Iustin Pop's avatar
Iustin Pop committed
529
                 , autobal::Bool)
Iustin Pop's avatar
Iustin Pop committed
530
531
532
533
    in
      case inst of
        Nothing -> False
        Just (_, i) ->
Iustin Pop's avatar
Iustin Pop committed
534
535
536
537
538
539
            Instance.name i == name &&
            Instance.vcpus i == vcpus &&
            Instance.mem i == mem &&
            Instance.pNode i == pdx &&
            Instance.sNode i == (if null snode
                                 then Node.noSecondary
540
                                 else sdx) &&
541
            Instance.autoBalance i == autobal &&
Iustin Pop's avatar
Iustin Pop committed
542
            isNothing fail1
Iustin Pop's avatar
Iustin Pop committed
543
544

prop_Text_Load_InstanceFail ktn fields =
Iustin Pop's avatar
Iustin Pop committed
545
546
    length fields /= 9 ==>
    case Text.loadInst nl fields of
Iustin Pop's avatar
Iustin Pop committed
547
548
      Types.Ok _ -> False
      Types.Bad msg -> "Invalid/incomplete instance data: '" `isPrefixOf` msg
549
    where nl = Data.Map.fromList ktn
Iustin Pop's avatar
Iustin Pop committed
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564

prop_Text_Load_Node name tm nm fm td fd tc fo =
    let conv v = if v < 0
                    then "?"
                    else show v
        tm_s = conv tm
        nm_s = conv nm
        fm_s = conv fm
        td_s = conv td
        fd_s = conv fd
        tc_s = conv tc
        fo_s = if fo
               then "Y"
               else "N"
        any_broken = any (< 0) [tm, nm, fm, td, fd, tc]
Iustin Pop's avatar
Iustin Pop committed
565
566
567
        gid = Group.uuid defGroup
    in case Text.loadNode defGroupAssoc
           [name, tm_s, nm_s, fm_s, td_s, fd_s, tc_s, fo_s, gid] of
Iustin Pop's avatar
Iustin Pop committed
568
569
570
571
         Nothing -> False
         Just (name', node) ->
             if fo || any_broken
             then Node.offline node
572
573
574
575
576
577
578
579
             else Node.name node == name' && name' == name &&
                  Node.alias node == name &&
                  Node.tMem node == fromIntegral tm &&
                  Node.nMem node == nm &&
                  Node.fMem node == fm &&
                  Node.tDsk node == fromIntegral td &&
                  Node.fDsk node == fd &&
                  Node.tCpu node == fromIntegral tc
Iustin Pop's avatar
Iustin Pop committed
580
581

prop_Text_Load_NodeFail fields =
Iustin Pop's avatar
Iustin Pop committed
582
    length fields /= 8 ==> isNothing $ Text.loadNode Data.Map.empty fields
Iustin Pop's avatar
Iustin Pop committed
583

584
prop_Text_NodeLSIdempotent node =
Iustin Pop's avatar
Iustin Pop committed
585
586
    (Text.loadNode defGroupAssoc.
         Utils.sepSplit '|' . Text.serializeNode defGroupList) n ==
587
588
589
590
    Just (Node.name n, n)
    -- override failN1 to what loadNode returns by default
    where n = node { Node.failN1 = True, Node.offline = False }

591
testText =
Iustin Pop's avatar
Iustin Pop committed
592
    [ run prop_Text_Load_Instance
Iustin Pop's avatar
Iustin Pop committed
593
594
595
    , run prop_Text_Load_InstanceFail
    , run prop_Text_Load_Node
    , run prop_Text_Load_NodeFail
596
    , run prop_Text_NodeLSIdempotent
597
598
    ]

599
-- ** Node tests
600

Iustin Pop's avatar
Iustin Pop committed
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
prop_Node_setAlias node name =
    Node.name newnode == Node.name node &&
    Node.alias newnode == name
    where _types = (node::Node.Node, name::String)
          newnode = Node.setAlias node name

prop_Node_setOffline node status =
    Node.offline newnode == status
    where newnode = Node.setOffline node status

prop_Node_setXmem node xm =
    Node.xMem newnode == xm
    where newnode = Node.setXmem node xm

prop_Node_setMcpu node mc =
    Node.mCpu newnode == mc
    where newnode = Node.setMcpu node mc

619
620
-- | Check that an instance add with too high memory or disk will be
-- rejected.
Iustin Pop's avatar
Iustin Pop committed
621
622
623
624
625
626
prop_Node_addPriFM node inst = Instance.mem inst >= Node.fMem node &&
                               not (Node.failN1 node)
                               ==>
                               case Node.addPri node inst'' of
                                 Types.OpFail Types.FailMem -> True
                                 _ -> False
627
    where _types = (node::Node.Node, inst::Instance.Instance)
Iustin Pop's avatar
Iustin Pop committed
628
629
630
631
632
633
634
635
636
637
638
639
640
          inst' = setInstanceSmallerThanNode node inst
          inst'' = inst' { Instance.mem = Instance.mem inst }

prop_Node_addPriFD node inst = Instance.dsk inst >= Node.fDsk node &&
                               not (Node.failN1 node)
                               ==>
                               case Node.addPri node inst'' of
                                 Types.OpFail Types.FailDisk -> True
                                 _ -> False
    where _types = (node::Node.Node, inst::Instance.Instance)
          inst' = setInstanceSmallerThanNode node inst
          inst'' = inst' { Instance.dsk = Instance.dsk inst }

Iustin Pop's avatar
Iustin Pop committed
641
642
643
644
645
prop_Node_addPriFC node inst (Positive extra) =
    not (Node.failN1 node) ==>
        case Node.addPri node inst'' of
          Types.OpFail Types.FailCPU -> True
          _ -> False
Iustin Pop's avatar
Iustin Pop committed
646
647
    where _types = (node::Node.Node, inst::Instance.Instance)
          inst' = setInstanceSmallerThanNode node inst
Iustin Pop's avatar
Iustin Pop committed
648
          inst'' = inst' { Instance.vcpus = Node.availCpu node + extra }
649

650
651
-- | Check that an instance add with too high memory or disk will be
-- rejected.
652
prop_Node_addSec node inst pdx =
653
654
    (Instance.mem inst >= (Node.fMem node - Node.rMem node) ||
     Instance.dsk inst >= Node.fDsk node) &&
Iustin Pop's avatar
Iustin Pop committed
655
    not (Node.failN1 node)
Iustin Pop's avatar
Iustin Pop committed
656
    ==> isFailure (Node.addSec node inst pdx)
657
        where _types = (node::Node.Node, inst::Instance.Instance, pdx::Int)
658

659
-- | Checks for memory reservation changes.
660
661
prop_Node_rMem inst =
    forAll (arbitrary `suchThat` ((> 0) . Node.fMem)) $ \node ->
662
663
    -- ab = auto_balance, nb = non-auto_balance
    -- we use -1 as the primary node of the instance
664
    let inst' = inst { Instance.pNode = -1, Instance.autoBalance = True }
665
        inst_ab = setInstanceSmallerThanNode node inst'
666
        inst_nb = inst_ab { Instance.autoBalance = False }
667
        -- now we have the two instances, identical except the
668
        -- autoBalance attribute
669
670
671
672
673
674
675
676
677
        orig_rmem = Node.rMem node
        inst_idx = Instance.idx inst_ab
        node_add_ab = Node.addSec node inst_ab (-1)
        node_add_nb = Node.addSec node inst_nb (-1)
        node_del_ab = liftM (flip Node.removeSec inst_ab) node_add_ab
        node_del_nb = liftM (flip Node.removeSec inst_nb) node_add_nb
    in case (node_add_ab, node_add_nb, node_del_ab, node_del_nb) of
         (Types.OpGood a_ab, Types.OpGood a_nb,
          Types.OpGood d_ab, Types.OpGood d_nb) ->
678
             printTestCase "Consistency checks failed" $
679
680
681
682
683
684
685
686
687
             Node.rMem a_ab >  orig_rmem &&
             Node.rMem a_ab - orig_rmem == Instance.mem inst_ab &&
             Node.rMem a_nb == orig_rmem &&
             Node.rMem d_ab == orig_rmem &&
             Node.rMem d_nb == orig_rmem &&
             -- this is not related to rMem, but as good a place to
             -- test as any
             inst_idx `elem` Node.sList a_ab &&
             not (inst_idx `elem` Node.sList d_ab)
688
689
         x -> printTestCase ("Failed to add/remove instances: " ++ show x)
              False
690

691
-- | Check mdsk setting.
Iustin Pop's avatar
Iustin Pop committed
692
693
694
695
696
prop_Node_setMdsk node mx =
    Node.loDsk node' >= 0 &&
    fromIntegral (Node.loDsk node') <= Node.tDsk node &&
    Node.availDisk node' >= 0 &&
    Node.availDisk node' <= Node.fDsk node' &&
Iustin Pop's avatar
Iustin Pop committed
697
698
    fromIntegral (Node.availDisk node') <= Node.tDsk node' &&
    Node.mDsk node' == mx'
Iustin Pop's avatar
Iustin Pop committed
699
700
701
702
703
704
705
    where _types = (node::Node.Node, mx::SmallRatio)
          node' = Node.setMdsk node mx'
          SmallRatio mx' = mx

-- Check tag maps
prop_Node_tagMaps_idempotent tags =
    Node.delTags (Node.addTags m tags) tags == m
706
    where m = Data.Map.empty
Iustin Pop's avatar
Iustin Pop committed
707
708
709
710

prop_Node_tagMaps_reject tags =
    not (null tags) ==>
    any (\t -> Node.rejectAddTags m [t]) tags
711
    where m = Node.addTags Data.Map.empty tags
Iustin Pop's avatar
Iustin Pop committed
712

Iustin Pop's avatar
Iustin Pop committed
713
714
715
716
717
prop_Node_showField node =
  forAll (elements Node.defaultFields) $ \ field ->
  fst (Node.showHeader field) /= Types.unknownField &&
  Node.showField node field /= Types.unknownField

718
719
720
721
722
723
724

prop_Node_computeGroups nodes =
  let ng = Node.computeGroups nodes
      onlyuuid = map fst ng
  in length nodes == sum (map (length . snd) ng) &&
     all (\(guuid, ns) -> all ((== guuid) . Node.group) ns) ng &&
     length (nub onlyuuid) == length onlyuuid &&
Iustin Pop's avatar
Iustin Pop committed
725
     (null nodes || not (null ng))
726

727
testNode =
Iustin Pop's avatar
Iustin Pop committed
728
729
730
731
732
    [ run prop_Node_setAlias
    , run prop_Node_setOffline
    , run prop_Node_setMcpu
    , run prop_Node_setXmem
    , run prop_Node_addPriFM
Iustin Pop's avatar
Iustin Pop committed
733
734
    , run prop_Node_addPriFD
    , run prop_Node_addPriFC
735
    , run prop_Node_addSec
736
    , run prop_Node_rMem
Iustin Pop's avatar
Iustin Pop committed
737
738
739
    , run prop_Node_setMdsk
    , run prop_Node_tagMaps_idempotent
    , run prop_Node_tagMaps_reject
Iustin Pop's avatar
Iustin Pop committed
740
    , run prop_Node_showField
741
    , run prop_Node_computeGroups
742
    ]
743
744


745
-- ** Cluster tests
746

747
748
-- | Check that the cluster score is close to zero for a homogeneous
-- cluster.
Iustin Pop's avatar
Iustin Pop committed
749
750
prop_Score_Zero node =
    forAll (choose (1, 1024)) $ \count ->
Iustin Pop's avatar
Iustin Pop committed
751
    (not (Node.offline node) && not (Node.failN1 node) && (count > 0) &&
752
     (Node.tDsk node > 0) && (Node.tMem node > 0)) ==>
753
    let fn = Node.buildPeers node Container.empty
Iustin Pop's avatar
Iustin Pop committed
754
        nlst = zip [1..] $ replicate count fn::[(Types.Ndx, Node.Node)]
Iustin Pop's avatar
Iustin Pop committed
755
        nl = Container.fromList nlst
756
757
758
        score = Cluster.compCV nl
    -- we can't say == 0 here as the floating point errors accumulate;
    -- this should be much lower than the default score in CLI.hs
Iustin Pop's avatar
Iustin Pop committed
759
    in score <= 1e-12
760

761
-- | Check that cluster stats are sane.
Iustin Pop's avatar
Iustin Pop committed
762
763
764
prop_CStats_sane node =
    forAll (choose (1, 1024)) $ \count ->
    (not (Node.offline node) && not (Node.failN1 node) &&
765
     (Node.availDisk node > 0) && (Node.availMem node > 0)) ==>
Iustin Pop's avatar
Iustin Pop committed
766
767
    let fn = Node.buildPeers node Container.empty
        nlst = zip [1..] $ replicate count fn::[(Types.Ndx, Node.Node)]
Iustin Pop's avatar
Iustin Pop committed
768
        nl = Container.fromList nlst
Iustin Pop's avatar
Iustin Pop committed
769
770
771
772
        cstats = Cluster.totalResources nl
    in Cluster.csAdsk cstats >= 0 &&
       Cluster.csAdsk cstats <= Cluster.csFdsk cstats

773
-- | Check that one instance is allocated correctly, without
774
-- rebalances needed.
775
776
777
778
779
780
781
782
783
784
prop_ClusterAlloc_sane node inst =
    forAll (choose (5, 20)) $ \count ->
    not (Node.offline node)
            && not (Node.failN1 node)
            && Node.availDisk node > 0
            && Node.availMem node > 0
            ==>
    let nl = makeSmallCluster node count
        il = Container.empty
        inst' = setInstanceSmallerThanNode node inst
785
786
    in case Cluster.genAllocNodes defGroupList nl 2 True >>=
       Cluster.tryAlloc nl il inst' of
787
         Types.Bad _ -> False
788
789
         Types.Ok as ->
             case Cluster.asSolutions as of
790
               [] -> False
791
               (xnl, xi, _, cv):[] ->
792
                   let il' = Container.add (Instance.idx xi) xi il
793
                       tbl = Cluster.Table xnl il' cv []
794
                   in not (canBalance tbl True True False)
795
796
797
798
               _ -> False

-- | Checks that on a 2-5 node cluster, we can allocate a random
-- instance spec via tiered allocation (whatever the original instance
799
-- spec), on either one or two nodes.
800
801
802
803
804
805
806
807
808
prop_ClusterCanTieredAlloc node inst =
    forAll (choose (2, 5)) $ \count ->
    forAll (choose (1, 2)) $ \rqnodes ->
    not (Node.offline node)
            && not (Node.failN1 node)
            && isNodeBig node 4
            ==>
    let nl = makeSmallCluster node count
        il = Container.empty
809
        allocnodes = Cluster.genAllocNodes defGroupList nl rqnodes True
Iustin Pop's avatar
Iustin Pop committed
810
811
    in case allocnodes >>= \allocnodes' ->
        Cluster.tieredAlloc nl il inst allocnodes' [] [] of
812
         Types.Bad _ -> False
813
814
815
         Types.Ok (_, _, il', ixes, cstats) -> not (null ixes) &&
                                      IntMap.size il' == length ixes &&
                                      length ixes == length cstats
816
817

-- | Checks that on a 4-8 node cluster, once we allocate an instance,
818
-- we can also evacuate it.
819
820
821
822
823
824
825
826
827
prop_ClusterAllocEvac node inst =
    forAll (choose (4, 8)) $ \count ->
    not (Node.offline node)
            && not (Node.failN1 node)
            && isNodeBig node 4
            ==>
    let nl = makeSmallCluster node count
        il = Container.empty
        inst' = setInstanceSmallerThanNode node inst
828
829
    in case Cluster.genAllocNodes defGroupList nl 2 True >>=
       Cluster.tryAlloc nl il inst' of
830
         Types.Bad _ -> False
831
832
         Types.Ok as ->
             case Cluster.asSolutions as of
833
               [] -> False
834
               (xnl, xi, _, _):[] ->
835
836
                   let sdx = Instance.sNode xi
                       il' = Container.add (Instance.idx xi) xi il
837
                   in case Cluster.tryEvac xnl il' [Instance.idx xi] [sdx] of
838
839
840
841
842
                        Just _ -> True
                        _ -> False
               _ -> False

-- | Check that allocating multiple instances on a cluster, then
843
-- adding an empty node, results in a valid rebalance.
844
845
prop_ClusterAllocBalance =
    forAll (genNode (Just 5) (Just 128)) $ \node ->
846
    forAll (choose (3, 5)) $ \count ->
847
    not (Node.offline node) && not (Node.failN1 node) ==>
848
849
850
    let nl = makeSmallCluster node count
        (hnode, nl') = IntMap.deleteFindMax nl
        il = Container.empty
851
        allocnodes = Cluster.genAllocNodes defGroupList nl' 2 True
852
        i_templ = createInstance Types.unitMem Types.unitDsk Types.unitCpu
Iustin Pop's avatar
Iustin Pop committed
853
854
    in case allocnodes >>= \allocnodes' ->
        Cluster.iterateAlloc nl' il i_templ allocnodes' [] [] of
855
         Types.Bad _ -> False
856
         Types.Ok (_, xnl, il', _, _) ->
857
858
859
                   let ynl = Container.add (Node.idx hnode) hnode xnl
                       cv = Cluster.compCV ynl
                       tbl = Cluster.Table ynl il' cv []
860
                   in canBalance tbl True True False
861

862
-- | Checks consistency.
863
864
865
prop_ClusterCheckConsistency node inst =
  let nl = makeSmallCluster node 3
      [node1, node2, node3] = Container.elems nl
Iustin Pop's avatar
Iustin Pop committed
866
      node3' = node3 { Node.group = 1 }
867
868
869
870
      nl' = Container.add (Node.idx node3') node3' nl
      inst1 = Instance.setBoth inst (Node.idx node1) (Node.idx node2)
      inst2 = Instance.setBoth inst (Node.idx node1) Node.noSecondary
      inst3 = Instance.setBoth inst (Node.idx node1) (Node.idx node3)
Iustin Pop's avatar
Iustin Pop committed
871
      ccheck = Cluster.findSplitInstances nl' . Container.fromList
872
873
874
875
  in null (ccheck [(0, inst1)]) &&
     null (ccheck [(0, inst2)]) &&
     (not . null $ ccheck [(0, inst3)])

876
-- | For now, we only test that we don't lose instances during the split.
877
878
879
880
881
882
883
884
885
prop_ClusterSplitCluster node inst =
  forAll (choose (0, 100)) $ \icnt ->
  let nl = makeSmallCluster node 2
      (nl', il') = foldl (\(ns, is) _ -> assignInstance ns is inst 0 1)
                   (nl, Container.empty) [1..icnt]
      gni = Cluster.splitCluster nl' il'
  in sum (map (Container.size . snd . snd) gni) == icnt &&
     all (\(guuid, (nl'', _)) -> all ((== guuid) . Node.group)
                                 (Container.elems nl'')) gni
886

887
testCluster =
888
    [ run prop_Score_Zero
Iustin Pop's avatar
Iustin Pop committed
889
    , run prop_CStats_sane
890
891
892
893
    , run prop_ClusterAlloc_sane
    , run prop_ClusterCanTieredAlloc
    , run prop_ClusterAllocEvac
    , run prop_ClusterAllocBalance
894
    , run prop_ClusterCheckConsistency
895
    , run prop_ClusterSplitCluster
896
    ]
Iustin Pop's avatar
Iustin Pop committed
897

898
-- ** OpCodes tests
Iustin Pop's avatar
Iustin Pop committed
899

900
-- | Check that opcode serialization is idempotent.
Iustin Pop's avatar
Iustin Pop committed
901
902
903
904
prop_OpCodes_serialization op =
  case J.readJSON (J.showJSON op) of
    J.Error _ -> False
    J.Ok op' -> op == op'
905
  where _types = op::OpCodes.OpCode
Iustin Pop's avatar
Iustin Pop committed
906
907
908
909

testOpCodes =
  [ run prop_OpCodes_serialization
  ]
Iustin Pop's avatar
Iustin Pop committed
910

911
912
913
-- ** Jobs tests

-- | Check that (queued) job\/opcode status serialization is idempotent.
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
prop_OpStatus_serialization os =
  case J.readJSON (J.showJSON os) of
    J.Error _ -> False
    J.Ok os' -> os == os'
  where _types = os::Jobs.OpStatus

prop_JobStatus_serialization js =
  case J.readJSON (J.showJSON js) of
    J.Error _ -> False
    J.Ok js' -> js == js'
  where _types = js::Jobs.JobStatus

testJobs =
  [ run prop_OpStatus_serialization
  , run prop_JobStatus_serialization
  ]

931
-- ** Loader tests
Iustin Pop's avatar
Iustin Pop committed
932
933

prop_Loader_lookupNode ktn inst node =
934
935
  Loader.lookupNode nl inst node == Data.Map.lookup node nl
  where nl = Data.Map.fromList ktn
Iustin Pop's avatar
Iustin Pop committed
936
937

prop_Loader_lookupInstance kti inst =
938
939
940
941
942
943
944
945
  Loader.lookupInstance il inst == Data.Map.lookup inst il
  where il = Data.Map.fromList kti

prop_Loader_assignIndices nodes =
  Data.Map.size nassoc == length nodes &&
  Container.size kt == length nodes &&
  (if not (null nodes)
   then maximum (IntMap.keys kt) == length nodes - 1
Iustin Pop's avatar
Iustin Pop committed
946
   else True)
947
  where (nassoc, kt) = Loader.assignIndices (map (\n -> (Node.name n, n)) nodes)
Iustin Pop's avatar
Iustin Pop committed
948
949

-- | Checks that the number of primary instances recorded on the nodes
950
-- is zero.
Iustin Pop's avatar
Iustin Pop committed
951
prop_Loader_mergeData ns =
Iustin Pop's avatar
Iustin Pop committed
952
  let na = Container.fromList $ map (\n -> (Node.idx n, n)) ns
953
  in case Loader.mergeData [] [] [] []
954
         (Loader.emptyCluster {Loader.cdNodes = na}) of
Iustin Pop's avatar
Iustin Pop committed
955
    Types.Bad _ -> False
956
    Types.Ok (Loader.ClusterData _ nl il _) ->
Iustin Pop's avatar
Iustin Pop committed
957
958
959
      let nodes = Container.elems nl
          instances = Container.elems il
      in (sum . map (length . Node.pList)) nodes == 0 &&
960
         null instances
Iustin Pop's avatar
Iustin Pop committed
961
962
963
964
965
966
967

testLoader =
  [ run prop_Loader_lookupNode
  , run prop_Loader_lookupInstance
  , run prop_Loader_assignIndices
  , run prop_Loader_mergeData
  ]
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986

-- ** Types tests

prop_AllocPolicy_serialisation apol =
    case Types.apolFromString (Types.apolToString apol) of
      Types.Ok p -> printTestCase ("invalid deserialisation " ++ show p) $
                    p == apol
      Types.Bad s -> printTestCase ("failed to deserialise: " ++ s) False

prop_DiskTemplate_serialisation dt =
    case Types.dtFromString (Types.dtToString dt) of
      Types.Ok p -> printTestCase ("invalid deserialisation " ++ show p) $
                    p == dt
      Types.Bad s -> printTestCase ("failed to deserialise: " ++ s) False

testTypes =
    [ run prop_AllocPolicy_serialisation
    , run prop_DiskTemplate_serialisation
    ]