Add new hail (shell) tests

Using some hand-crafted files (manually modified from a real cluster
running master branch), we test that hail behaves as we expect:
doesn't load obvious wrong data, can do relocations/allocations, etc.
Signed-off-by: Iustin Pop <iustin@google.com>
Reviewed-by: René Nussbaumer <rn@google.com>

Makefile.am

@@ -86,6 +86,7 @@ DIRS = \
 	qa \
 	test \
 	test/data \
+	test/data/htools \
 	test/data/ovfdata \
 	test/data/ovfdata/other \
 	tools
@@ -710,6 +711,10 @@ TEST_FILES = \
 	test/data/kvm_0.12.5_help.txt \
 	test/data/kvm_0.9.1_help.txt \
 	test/data/sys_drbd_usermode_helper.txt \
+	test/data/htools/hail-alloc-drbd.json \
+	test/data/htools/hail-change-group.json \
+	test/data/htools/hail-node-evac.json \
+	test/data/htools/hail-reloc-drbd.json \
 	test/data/ovfdata/compr_disk.vmdk.gz \
 	test/data/ovfdata/config.ini \
 	test/data/ovfdata/corrupted_resources.ovf \

htools/cli-tests-defs.sh

@@ -21,6 +21,8 @@
 
 HBINARY=${HBINARY:-./htools/htools-hpc}
 
+TESTDATA_DIR=${TOP_SRCDIR:-.}/test/data/htools
+
 hbal() {
   HTOOLS=hbal $HBINARY "$@"
 }

htools/offline-test.sh

@@ -101,7 +101,7 @@ echo Checking rebalancing
 # policy, then we change all nodes from this group to the allocable
 # one, and we check for rebalancing
 FROOT="$T/simu-rebal-orig"
-hspace --simu p,4,8T,64g,16 --simu u,4,8T,64g,16 \
+hspace --simu u,4,8T,64g,16 --simu p,4,8T,64g,16 \
   -S $FROOT --disk-template drbd -l 8 >/dev/null 2>&1
 for suffix in standard tiered; do
   RELOC="$T/simu-rebal-merged.$suffix"
@@ -129,4 +129,48 @@ for suffix in standard tiered; do
 done
 echo OK
 
+echo IAllocator checks
+# test that on invalid files it can't parse the request
+! hail /dev/null 2>&1 | grep -q "Invalid JSON"
+! hail <(echo '[]') >/dev/null 2>&1
+! hail <(echo '{}') 2>&1 | grep -q "key 'request' not found"
+! hail <(echo '{"request": 0}') 2>&1 | grep -q "key 'request'"
+! hail $TESTDATA_DIR/hail-invalid-reloc.json >/dev/null 2>&1
+
+# just test that it can read the file, print the cluster and generate
+# pre and post allocation files
+hail -v -v -v -p $TESTDATA_DIR/hail-alloc-drbd.json \
+  -S $T/hail-alloc >/dev/null 2>&1
+! cmp -s $T/hail-alloc.pre-ialloc $T/hail-alloc.post-ialloc
+hail -v -v -v -p $TESTDATA_DIR/hail-reloc-drbd.json \
+  -S $T/hail-reloc >/dev/null 2>&1
+! cmp -s $T/hail-reloc.pre-ialloc $T/hail-reloc.post-ialloc
+
+# and now start the real tests
+hail $TESTDATA_DIR/hail-alloc-drbd.json | \
+  grep -q '"success":true,.*,"result":\["node2","node1"\]'
+hail $TESTDATA_DIR/hail-reloc-drbd.json | \
+  grep -q '"success":true,.*,"result":\["node1"\]'
+
+hail $TESTDATA_DIR/hail-node-evac.json | \
+  grep -Fq '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
+
+hail $TESTDATA_DIR/hail-change-group.json | \
+  grep -Fq '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
+
+for evac_mode in primary-only secondary-only all; do
+  sed -e 's/"evac_mode": "all"/"evac_mode": "'${evac_mode}'"/' \
+    < $TESTDATA_DIR/hail-node-evac.json \
+    > $T/hail-node-evac.json.$evac_mode
+  hail $T/hail-node-evac.json.$evac_mode | \
+    grep -q '"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"'
+done
+
+# check that hail can use the simu and text backends
+hail --simu p,8,8T,16g,16 $TESTDATA_DIR/hail-alloc-drbd.json | \
+  grep -q '"success":true,'
+hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json | \
+  grep -q '"success":true,'
+echo OK
+
 echo All OK

test/data/htools/hail-alloc-drbd.json

+{
+  "cluster_tags": [
+    "htools:iextags:test",
+    "htools:iextags:service-group"
+  ],
+  "nodegroups": {
+    "uuid-group-1": {
+      "ipolicy": {
+        "std": {
+          "nic-count": 1,
+          "disk-size": 1024,
+          "disk-count": 1,
+          "memory-size": 128,
+          "cpu-count": 1,
+          "spindle-use": 1
+        },
+        "min": {
+          "nic-count": 1,
+          "disk-size": 128,
+          "disk-count": 1,
+          "memory-size": 128,
+          "cpu-count": 1,
+          "spindle-use": 1
+        },
+        "max": {
+          "nic-count": 8,
+          "disk-size": 1048576,
+          "disk-count": 16,
+          "memory-size": 32768,
+          "cpu-count": 8,
+          "spindle-use": 8
+        },
+        "vcpu-ratio": 4.0,
+        "disk-templates": [
+          "sharedfile",
+          "diskless",
+          "plain",
+          "blockdev",
+          "drbd",
+          "file",
+          "rbd"
+        ],
+        "spindle-ratio": 32.0
+      },
+      "alloc_policy": "preferred",
+      "name": "default"
+    }
+  },
+  "ipolicy": {
+    "std": {
+      "nic-count": 1,
+      "disk-size": 1024,
+      "memory-size": 128,
+      "cpu-count": 1,
+      "disk-count": 1,
+      "spindle-use": 1
+    },
+    "min": {
+      "nic-count": 1,
+      "disk-size": 1024,
+      "memory-size": 128,
+      "cpu-count": 1,
+      "disk-count": 1,
+      "spindle-use": 1
+    },
+    "max": {
+      "nic-count": 8,
+      "disk-size": 1048576,
+      "memory-size": 32768,
+      "cpu-count": 8,
+      "disk-count": 16,
+      "spindle-use": 8
+    },
+    "vcpu-ratio": 4.0,
+    "disk-templates": [
+      "sharedfile",
+      "diskless",
+      "plain",
+      "blockdev",
+      "drbd",
+      "file",
+      "rbd"
+    ],
+    "spindle-ratio": 32.0
+  },
+  "enabled_hypervisors": [
+    "xen-pvm",
+    "xen-hvm"
+  ],
+  "cluster_name": "cluster",
+  "instances": {
+    "instance14": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 256,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:eb:0b:a5",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "admin_state": "down",
+      "disk_template": "drbd",
+      "spindle_use": 1,
+      "memory": 128,
+      "nodes": [
+        "node3",
+        "node4"
+      ],
+      "os": "debian-image"
+    },
+    "instance13": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 512
+        }
+      ],
+      "disk_space_total": 512,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:7f:8c:9c",
+          "link": "xen-br1",
+          "mode": "bridged",
+          "bridge": "xen-br1"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "plain",
+      "memory": 128,
+      "nodes": [
+        "node4"
+      ],
+      "os": "instance-debootstrap"
+    },
+    "instance18": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 128,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:55:94:93",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "plain",
+      "memory": 8192,
+      "nodes": [
+        "node4"
+      ],
+      "os": "instance-debootstrap"
+    },
+    "instance19": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 256,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:15:92:6f",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "drbd",
+      "memory": 128,
+      "nodes": [
+        "node3",
+        "node4"
+      ],
+      "os": "debian-image"
+    },
+    "instance2": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 256,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:73:20:3e",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "up",
+      "disk_template": "drbd",
+      "memory": 128,
+      "nodes": [
+        "node3",
+        "node4"
+      ],
+      "os": "debian-image"
+    },
+    "instance3": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 256
+        },
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 384,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:ec:e8:a2",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "plain",
+      "memory": 128,
+      "nodes": [
+        "node4"
+      ],
+      "os": "debian-image"
+    },
+    "instance4": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 2048
+        }
+      ],
+      "disk_space_total": 2176,
+      "hypervisor": "xen-pvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:62:b0:76",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "drbd",
+      "memory": 128,
+      "nodes": [
+        "node4",
+        "node3"
+      ],
+      "os": "instance-debootstrap"
+    },
+    "instance8": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 256
+        }
+      ],
+      "disk_space_total": 256,
+      "hypervisor": "kvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:3f:6d:e3",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "plain",
+      "memory": 128,
+      "nodes": [
+        "node4"
+      ],
+      "os": "debian-image"
+    },
+    "instance9": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 128
+        }
+      ],
+      "disk_space_total": 256,
+      "hypervisor": "xen-pvm",
+      "tags": [
+        "test:test"
+      ],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:10:d2:01",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "drbd",
+      "memory": 128,
+      "nodes": [
+        "node3",
+        "node4"
+      ],
+      "os": "instance-debootstrap"
+    },
+    "instance20": {
+      "disks": [
+        {
+          "mode": "rw",
+          "size": 512
+        }
+      ],
+      "disk_space_total": 512,
+      "hypervisor": "kvm",
+      "tags": [],
+      "nics": [
+        {
+          "ip": null,
+          "mac": "aa:00:00:db:2a:6d",
+          "link": "xen-br0",
+          "mode": "bridged",
+          "bridge": "xen-br0"
+        }
+      ],
+      "vcpus": 1,
+      "spindle_use": 1,
+      "admin_state": "down",
+      "disk_template": "plain",
+      "memory": 128,
+      "nodes": [
+        "node4"
+      ],
+      "os": "instance-debootstrap"
+    }
+  },
+  "version": 2,
+  "nodes": {
+    "node1": {
+      "total_disk": 1377280,
+      "total_cpus": 4,
+      "group": "uuid-group-1",
+      "secondary_ip": "192.168.2.1",
+      "i_pri_up_memory": 0,
+      "tags": [],
+      "master_candidate": true,
+      "free_memory": 31389,
+      "ndparams": {
+        "spindle_count": 1,
+        "oob_program": null
+      },
+      "reserved_memory": 1017,
+      "master_capable": true,
+      "free_disk": 1377280,
+      "drained": false,
+      "total_memory": 32763,
+      "primary_ip": "192.168.1.1",
+      "i_pri_memory": 0,
+      "vm_capable": true,
+      "offline": false
+    },
+    "node2": {
+      "total_disk": 1377280,
+      "total_cpus": 4,
+      "group": "uuid-group-1",
+      "secondary_ip": "192.168.2.2",
+      "i_pri_up_memory": 0,
+      "tags": [],
+      "master_candidate": true,
+      "free_memory": 31746,
+      "ndparams": {
+        "spindle_count": 1,
+        "oob_program": null
+      },
+      "reserved_memory": 1017,
+      "master_capable": true,
+      "free_disk": 1376640,
+      "drained": false,
+      "total_memory": 32763,
+      "primary_ip": "192.168.1.2",
+      "i_pri_memory": 0,
+      "vm_capable": true,
+      "offline": false
+    },
+    "node3": {
+      "total_disk": 1377304,
+      "total_cpus": 4,
+      "group": "uuid-group-1",
+      "secondary_ip": "192.168.2.3",
+      "i_pri_up_memory": 128,
+      "tags": [],
+      "master_candidate": true,
+      "free_memory": 31234,
+      "ndparams": {
+        "spindle_count": 1,
+        "oob_program": null
+      },
+      "reserved_memory": 1017,
+      "master_capable": true,
+      "free_disk": 1373336,
+      "drained": false,
+      "total_memory": 32763,
+      "primary_ip": "192.168.1.3",
+      "i_pri_memory": 2432,
+      "vm_capable": true,
+      "offline": false
+    },
+    "node4": {
+      "total_disk": 1377280,
+      "total_cpus": 4,
+      "group": "uuid-group-1",
+      "secondary_ip": "192.168.2.4",
+      "i_pri_up_memory": 128,
+      "tags": [],
+      "master_candidate": true,
+      "free_memory": 22914,
+      "ndparams": {
+        "spindle_count": 1,
+        "oob_program": null
+      },
+      "reserved_memory": 1017,
+      "master_capable": true,
+      "free_disk": 1371520,
+      "drained": false,
+      "total_memory": 32763,
+      "primary_ip": "192.168.1.4",
+      "i_pri_memory": 23552,
+      "vm_capable": true,
+      "offline": false
+    }
+  },
+  "request": {
+    "disks": [
+      {
+        "mode": "rw",
+        "size": 1024
+      }
+    ],
+    "required_nodes": 2,
+    "name": "instance1",
+    "tags": [],
+    "hypervisor": "xen-pvm",
+    "disk_space_total": 1024,
+    "nics": [
+      {
+        "ip": null,
+        "mac": "00:11:22:33:44:55",
+        "bridge": null
+      }
+    ],
+    "vcpus": 1,
+    "spindle_use": 1,
+    "os": "instance-debootstrap",
+    "disk_template": "drbd",
+    "memory": 1024,
+    "type": "allocate"
+  }
+}

test/data/htools/hail-change-group.json

+{
+  "cluster_tags": [
+    "htools:iextags:test",
+    "htools:iextags:service-group"
+  ],
+  "nodegroups": {