QA: Assert equality of ssconf_master_candidate_certs

In order to hunt down some flakiness, this patch adds
and additional check to the QA of renew-crypto to ensure
the state of ssconf_master_candidate_certs.
Signed-off-by: Helga Velroyen <helgav@google.com>
Reviewed-by: Klaus Aehlig <aehlig@google.com>

qa/qa_cluster.py

@@ -1150,6 +1150,37 @@ def TestClusterVersion():
   AssertCommand(["gnt-cluster", "version"])
 
 
+def _AssertSsconfCertFiles(master):
+  """This asserts that all ssconf_master_candidate_certs have the same content.
+
+  @type master: string
+  @param master: name of the master node
+
+  """
+  nodes = qa_config.get("nodes")
+  ssconf_file = "/var/lib/ganeti/ssconf_master_candidates_certs"
+  ssconf_content = {}
+  for node in nodes:
+    cmd = ["cat", ssconf_file]
+    print "Ssconf Master Certificates of node '%s'." % node.primary
+    result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
+    ssconf_content[node] = result_output
+
+    # Clean up result to make it comparable:
+    # remove trailing whitespace from each line, remove empty lines, sort lines
+    lines_node = result_output.split('\n')
+    lines_node = [line.strip() for line in lines_node if len(line.strip()) > 0]
+    lines_node.sort()
+
+    ssconf_content[node] = lines_node
+
+  first_node = nodes[0]
+  for node in nodes[1:]:
+    if not ssconf_content[node] == ssconf_content[first_node]:
+      raise Exception("Cert list of node '%s' differs from the list of node"
+                      " '%s'." % (node, first_node))
+
+
 def TestClusterRenewCrypto():
   """gnt-cluster renew-crypto"""
   master = qa_config.GetMasterNode()
@@ -1205,16 +1236,19 @@ def TestClusterRenewCrypto():
                    "--new-cluster-certificate", "--new-confd-hmac-key",
                    "--new-rapi-certificate", "--new-cluster-domain-secret",
                    "--new-node-certificates"])
+    _AssertSsconfCertFiles(master)
     AssertCommand(["gnt-cluster", "verify"])
 
     # Only renew node certificates
     AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                    "--new-node-certificates"])
+    _AssertSsconfCertFiles(master)
     AssertCommand(["gnt-cluster", "verify"])
 
     # Restore RAPI certificate
     AssertCommand(["gnt-cluster", "renew-crypto", "--force",
                    "--rapi-certificate=%s" % rapi_cert_backup])
+    _AssertSsconfCertFiles(master)
     AssertCommand(["gnt-cluster", "verify"])
   finally:
     AssertCommand(["rm", "-f", rapi_cert_backup])