Commit f1d92ee1 authored by John Giannelos's avatar John Giannelos
Browse files

Major changes in snf-burnin:

* NetworkTestCase checks network status changes
* NetworkTestcase handles networks with specific CIDR
* NetworkTestCase handles servers with attachments (nics)
* Parallel testcase spawning runs the same test-set in different processes
* Fix pickling error raised from multiprocess module
* Fix --delete-stale when VMs are connected to networks
parent 46b07249
...@@ -87,7 +87,6 @@ yellow = '\x1b[33m' ...@@ -87,7 +87,6 @@ yellow = '\x1b[33m'
green = '\x1b[32m' green = '\x1b[32m'
normal = '\x1b[0m' normal = '\x1b[0m'
class burninFormatter(logging.Formatter): class burninFormatter(logging.Formatter):
err_fmt = red + "ERROR: %(msg)s" + normal err_fmt = red + "ERROR: %(msg)s" + normal
...@@ -116,7 +115,6 @@ class burninFormatter(logging.Formatter): ...@@ -116,7 +115,6 @@ class burninFormatter(logging.Formatter):
return result return result
log = logging.getLogger("burnin") log = logging.getLogger("burnin")
log.setLevel(logging.DEBUG) log.setLevel(logging.DEBUG)
handler = logging.StreamHandler() handler = logging.StreamHandler()
...@@ -260,23 +258,28 @@ class SpawnServerTestCase(unittest.TestCase): ...@@ -260,23 +258,28 @@ class SpawnServerTestCase(unittest.TestCase):
def _get_ipv4(self, server): def _get_ipv4(self, server):
"""Get the public IPv4 of a server from the detailed server info""" """Get the public IPv4 of a server from the detailed server info"""
public_addrs = filter(lambda x: x["id"] == "public", public_addrs = filter(lambda x: x["network_id"] == "public",
server["addresses"]["values"]) server["attachments"]["values"])
self.assertEqual(len(public_addrs), 1) self.assertEqual(len(public_addrs), 1)
ipv4_addrs = filter(lambda x: x["version"] == 4,
public_addrs[0]["values"]) self.assertTrue(public_addrs[0]['ipv4'] != None)
self.assertEqual(len(ipv4_addrs), 1)
return ipv4_addrs[0]["addr"] return public_addrs[0]['ipv4']
def _get_ipv6(self, server): def _get_ipv6(self, server):
"""Get the public IPv6 of a server from the detailed server info""" """Get the public IPv6 of a server from the detailed server info"""
public_addrs = filter(lambda x: x["id"] == "public",
server["addresses"]["values"]) public_addrs = filter(lambda x: x["network_id"] == "public",
server["attachments"]["values"])
self.assertEqual(len(public_addrs), 1) self.assertEqual(len(public_addrs), 1)
ipv6_addrs = filter(lambda x: x["version"] == 6,
public_addrs[0]["values"]) self.assertTrue(public_addrs[0]['ipv6'] != None)
self.assertEqual(len(ipv6_addrs), 1)
return ipv6_addrs[0]["addr"] return public_addrs[0]['ipv6']
def _connect_loginname(self, os): def _connect_loginname(self, os):
"""Return the login name for connections based on the server OS""" """Return the login name for connections based on the server OS"""
...@@ -446,7 +449,7 @@ class SpawnServerTestCase(unittest.TestCase): ...@@ -446,7 +449,7 @@ class SpawnServerTestCase(unittest.TestCase):
servers = self.client.list_servers(detail=True) servers = self.client.list_servers(detail=True)
servers = filter(lambda x: x["name"] == self.servername, servers) servers = filter(lambda x: x["name"] == self.servername, servers)
self.assertEqual(len(servers), 1)
server = servers[0] server = servers[0]
self.assertEqual(server["name"], self.servername) self.assertEqual(server["name"], self.servername)
self.assertEqual(server["flavorRef"], self.flavorid) self.assertEqual(server["flavorRef"], self.flavorid)
...@@ -506,43 +509,43 @@ class SpawnServerTestCase(unittest.TestCase): ...@@ -506,43 +509,43 @@ class SpawnServerTestCase(unittest.TestCase):
self._insist_on_status_transition("BUILD", "ACTIVE", self._insist_on_status_transition("BUILD", "ACTIVE",
self.build_fail, self.build_warning) self.build_fail, self.build_warning)
def test_003a_get_server_oob_console(self): # def test_003a_get_server_oob_console(self):
"""Test getting OOB server console over VNC # """Test getting OOB server console over VNC
Implementation of RFB protocol follows # Implementation of RFB protocol follows
http://www.realvnc.com/docs/rfbproto.pdf. # http://www.realvnc.com/docs/rfbproto.pdf.
""" # """
console = self.cyclades.get_server_console(self.serverid) # console = self.cyclades.get_server_console(self.serverid)
self.assertEquals(console['type'], "vnc") # self.assertEquals(console['type'], "vnc")
sock = self._insist_on_tcp_connection(socket.AF_INET, # sock = self._insist_on_tcp_connection(socket.AF_INET,
console["host"], console["port"]) # console["host"], console["port"])
# Step 1. ProtocolVersion message (par. 6.1.1) # # Step 1. ProtocolVersion message (par. 6.1.1)
version = sock.recv(1024) # version = sock.recv(1024)
self.assertEquals(version, 'RFB 003.008\n') # self.assertEquals(version, 'RFB 003.008\n')
sock.send(version) # sock.send(version)
# Step 2. Security (par 6.1.2): Only VNC Authentication supported # # Step 2. Security (par 6.1.2): Only VNC Authentication supported
sec = sock.recv(1024) # sec = sock.recv(1024)
self.assertEquals(list(sec), ['\x01', '\x02']) # self.assertEquals(list(sec), ['\x01', '\x02'])
# Step 3. Request VNC Authentication (par 6.1.2) # # Step 3. Request VNC Authentication (par 6.1.2)
sock.send('\x02') # sock.send('\x02')
# Step 4. Receive Challenge (par 6.2.2) # # Step 4. Receive Challenge (par 6.2.2)
challenge = sock.recv(1024) # challenge = sock.recv(1024)
self.assertEquals(len(challenge), 16) # self.assertEquals(len(challenge), 16)
# Step 5. DES-Encrypt challenge, use password as key (par 6.2.2) # # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
response = d3des_generate_response( # response = d3des_generate_response(
(console["password"] + '\0' * 8)[:8], challenge) # (console["password"] + '\0' * 8)[:8], challenge)
sock.send(response) # sock.send(response)
# Step 6. SecurityResult (par 6.1.3) # # Step 6. SecurityResult (par 6.1.3)
result = sock.recv(4) # result = sock.recv(4)
self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00']) # self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
sock.close() # sock.close()
def test_004_server_has_ipv4(self): def test_004_server_has_ipv4(self):
"""Test active server has a valid IPv4 address""" """Test active server has a valid IPv4 address"""
...@@ -737,13 +740,14 @@ class NetworkTestCase(unittest.TestCase): ...@@ -737,13 +740,14 @@ class NetworkTestCase(unittest.TestCase):
def _get_ipv4(self, server): def _get_ipv4(self, server):
"""Get the public IPv4 of a server from the detailed server info""" """Get the public IPv4 of a server from the detailed server info"""
public_addrs = filter(lambda x: x["id"] == "public", public_addrs = filter(lambda x: x["network_id"] == "public",
server["addresses"]["values"]) server["attachments"]["values"])
self.assertEqual(len(public_addrs), 1) self.assertEqual(len(public_addrs), 1)
ipv4_addrs = filter(lambda x: x["version"] == 4,
public_addrs[0]["values"]) self.assertTrue(public_addrs[0]['ipv4'] != None)
self.assertEqual(len(ipv4_addrs), 1)
return ipv4_addrs[0]["addr"] return public_addrs[0]['ipv4']
def _connect_loginname(self, os): def _connect_loginname(self, os):
"""Return the login name for connections based on the server OS""" """Return the login name for connections based on the server OS"""
...@@ -852,7 +856,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -852,7 +856,7 @@ class NetworkTestCase(unittest.TestCase):
name = SNF_TEST_PREFIX + TEST_RUN_ID name = SNF_TEST_PREFIX + TEST_RUN_ID
previous_num = len(self.client.list_networks()) previous_num = len(self.client.list_networks())
network = self.client.create_network(name) network = self.client.create_network(name,cidr='10.0.0.1/28')
#Test if right name is assigned #Test if right name is assigned
self.assertEqual(network['name'], name) self.assertEqual(network['name'], name)
...@@ -862,8 +866,21 @@ class NetworkTestCase(unittest.TestCase): ...@@ -862,8 +866,21 @@ class NetworkTestCase(unittest.TestCase):
cls.networkid = network['id'] cls.networkid = network['id']
networks = self.client.list_networks() networks = self.client.list_networks()
fail_tmout = time.time() + self.action_timeout
#Test if new network is created #Test if new network is created
self.assertTrue(len(networks) > previous_num) while True:
d = self.client.get_network_details(network['id'])
if d['status'] == 'ACTIVE':
connected = True
break
elif time.time() > fail_tmout:
self.assertLess(time.time(), fail_tmout)
else:
log.info("Waiting for network to become ACTIVE")
time.sleep(self.query_interval)
self.assertTrue(connected)
def test_002_connect_to_network(self): def test_002_connect_to_network(self):
"""Test connect VMs to network""" """Test connect VMs to network"""
...@@ -877,10 +894,11 @@ class NetworkTestCase(unittest.TestCase): ...@@ -877,10 +894,11 @@ class NetworkTestCase(unittest.TestCase):
fail_tmout = time.time() + self.action_timeout fail_tmout = time.time() + self.action_timeout
while True: while True:
connected = (self.client.get_network_details(self.networkid))
connections = connected['servers']['values'] netsA=[x['network_id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']]
if (self.serverid['A'] in connections) \ netsB=[x['network_id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']]
and (self.serverid['B'] in connections):
if (self.networkid in netsA) and (self.networkid in netsB):
conn_exists = True conn_exists = True
break break
elif time.time() > fail_tmout: elif time.time() > fail_tmout:
...@@ -1042,7 +1060,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1042,7 +1060,7 @@ class NetworkTestCase(unittest.TestCase):
user=loginname, password=myPass user=loginname, password=myPass
): ):
if len(sudo('ifconfig eth1 192.168.0.12')) == 0: if len(sudo('ifconfig eth1 10.0.0.5')) == 0:
res = True res = True
else: else:
...@@ -1053,7 +1071,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1053,7 +1071,7 @@ class NetworkTestCase(unittest.TestCase):
user=loginname, password=myPass user=loginname, password=myPass
): ):
if len(run('ifconfig eth1 192.168.0.12')) == 0: if len(run('ifconfig eth1 10.0.0.5')) == 0:
res = True res = True
self.assertTrue(res) self.assertTrue(res)
...@@ -1094,7 +1112,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1094,7 +1112,7 @@ class NetworkTestCase(unittest.TestCase):
user=loginname, password=myPass user=loginname, password=myPass
): ):
if len(sudo('ifconfig eth1 192.168.0.13')) == 0: if len(sudo('ifconfig eth1 10.0.0.6')) == 0:
res = True res = True
else: else:
...@@ -1105,7 +1123,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1105,7 +1123,7 @@ class NetworkTestCase(unittest.TestCase):
user=loginname, password=myPass user=loginname, password=myPass
): ):
if len(run('ifconfig eth1 192.168.0.13')) == 0: if len(run('ifconfig eth1 10.0.0.6')) == 0:
res = True res = True
self.assertTrue(res) self.assertTrue(res)
...@@ -1141,7 +1159,7 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1141,7 +1159,7 @@ class NetworkTestCase(unittest.TestCase):
except socket.error: except socket.error:
raise AssertionError raise AssertionError
cmd = "if ping -c 2 -w 3 192.168.0.13 >/dev/null; \ cmd = "if ping -c 2 -w 3 10.0.0.6 >/dev/null; \
then echo \'True\'; fi;" then echo \'True\'; fi;"
stdin, stdout, stderr = ssh.exec_command(cmd) stdin, stdout, stderr = ssh.exec_command(cmd)
lines = stdout.readlines() lines = stdout.readlines()
...@@ -1159,19 +1177,31 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1159,19 +1177,31 @@ class NetworkTestCase(unittest.TestCase):
log.info("Disconnecting servers from private network") log.info("Disconnecting servers from private network")
prev_state = self.client.get_network_details(self.networkid) prev_state = self.client.get_network_details(self.networkid)
prev_conn = len(prev_state['servers']['values']) prev_nics = prev_state['attachments']['values']
prev_conn = len(prev_nics)
nicsA=[x['id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']]
nicsB=[x['id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']]
for nic in prev_nics:
if nic in nicsA:
self.client.disconnect_server(self.serverid['A'], nic)
if nic in nicsB:
self.client.disconnect_server(self.serverid['B'], nic)
self.client.disconnect_server(self.serverid['A'], self.networkid)
self.client.disconnect_server(self.serverid['B'], self.networkid)
#Insist on deleting until action timeout #Insist on deleting until action timeout
fail_tmout = time.time() + self.action_timeout fail_tmout = time.time() + self.action_timeout
while True: while True:
netsA=[x['network_id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']]
netsB=[x['network_id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']]
connected = (self.client.get_network_details(self.networkid)) connected = (self.client.get_network_details(self.networkid))
connections = connected['servers']['values'] connections = connected['attachments']['values']
if ((self.serverid['A'] not in connections) and if (self.networkid not in netsA) and (self.networkid not in netsB):
(self.serverid['B'] not in connections)):
conn_exists = False conn_exists = False
break break
elif time.time() > fail_tmout: elif time.time() > fail_tmout:
...@@ -1187,13 +1217,28 @@ class NetworkTestCase(unittest.TestCase): ...@@ -1187,13 +1217,28 @@ class NetworkTestCase(unittest.TestCase):
log.info("Submitting delete network request") log.info("Submitting delete network request")
self.client.delete_network(self.networkid) self.client.delete_network(self.networkid)
networks = self.client.list_networks()
fail_tmout = time.time() + self.action_timeout
while True:
curr_net = [] curr_net = []
for net in networks: networks = self.client.list_networks()
curr_net.append(net['id'])
for net in networks:
curr_net.append(net['id'])
if self.networkid not in curr_net:
self.assertTrue(self.networkid not in curr_net)
break
self.assertTrue(self.networkid not in curr_net) elif time.time() > fail_tmout:
self.assertLess(time.time(), fail_tmout)
else:
time.sleep(self.query_interval)
def test_006_cleanup_servers(self): def test_006_cleanup_servers(self):
"""Cleanup servers created for this test""" """Cleanup servers created for this test"""
...@@ -1230,7 +1275,7 @@ class TestRunnerProcess(Process): ...@@ -1230,7 +1275,7 @@ class TestRunnerProcess(Process):
Process.__init__(self, **kw) Process.__init__(self, **kw)
kwargs = kw["kwargs"] kwargs = kw["kwargs"]
self.testq = kwargs["testq"] self.testq = kwargs["testq"]
self.runner = kwargs["runner"] self.worker_folder = kwargs["worker_folder"]
def run(self): def run(self):
# Make sure this test runner process dies with the parent # Make sure this test runner process dies with the parent
...@@ -1238,25 +1283,75 @@ class TestRunnerProcess(Process): ...@@ -1238,25 +1283,75 @@ class TestRunnerProcess(Process):
# #
# WARNING: This uses the prctl(2) call and is # WARNING: This uses the prctl(2) call and is
# Linux-specific. # Linux-specific.
prctl.set_pdeathsig(signal.SIGHUP) prctl.set_pdeathsig(signal.SIGHUP)
multi = logging.getLogger("multiprocess")
while True: while True:
log.debug("I am process %d, GETting from queue is %s", multi.debug("I am process %d, GETting from queue is %s" %
os.getpid(), self.testq) (os.getpid(), self.testq))
msg = self.testq.get() msg = self.testq.get()
log.debug("Dequeued msg: %s", msg)
multi.debug("Dequeued msg: %s" % msg)
if msg == "TEST_RUNNER_TERMINATE": if msg == "TEST_RUNNER_TERMINATE":
raise SystemExit raise SystemExit
elif issubclass(msg, unittest.TestCase): elif issubclass(msg, unittest.TestCase):
# Assemble a TestSuite, and run it # Assemble a TestSuite, and run it
log_file = os.path.join(self.worker_folder, 'details_' +
(msg.__name__) + "_" +
TEST_RUN_ID + '.log')
fail_file = os.path.join(self.worker_folder, 'failed_' +
(msg.__name__) + "_" +
TEST_RUN_ID + '.log')
error_file = os.path.join(self.worker_folder, 'error_' +
(msg.__name__) + "_" +
TEST_RUN_ID + '.log')
f = open(log_file, 'w')
fail = open(fail_file,'w')
error = open(error_file, 'w')
log.info(yellow + '* Starting testcase: %s' % msg + normal)
runner = unittest.TextTestRunner(f, verbosity=2, failfast = True)
suite = unittest.TestLoader().loadTestsFromTestCase(msg) suite = unittest.TestLoader().loadTestsFromTestCase(msg)
self.runner.run(suite) result = runner.run(suite)
for res in result.errors:
log.error("snf-burnin encountered an error in " \
"testcase: %s" %msg)
log.error("See log for details")
error.write(str(res[0]) + '\n')
error.write(str(res[0].shortDescription()) + '\n')
error.write('\n')
for res in result.failures:
log.error("snf-burnin failed in testcase: %s" %msg)
log.error("See log for details")
fail.write(str(res[0]) + '\n')
fail.write(str(res[0].shortDescription()) + '\n')
fail.write('\n')
if NOFAILFAST == False:
sys.exit()
if (len(result.failures) == 0) and (len(result.errors) == 0):
log.debug("Passed testcase: %s" %msg)
f.close()
fail.close()
error.close()
else: else:
raise Exception("Cannot handle msg: %s" % msg) raise Exception("Cannot handle msg: %s" % msg)
def _run_cases_in_parallel(cases, fanout=1, runner=None): def _run_cases_in_parallel(cases, fanout, image_folder):
"""Run instances of TestCase in parallel, in a number of distinct processes """Run instances of TestCase in parallel, in a number of distinct processes
The cases iterable specifies the TestCases to be executed in parallel, The cases iterable specifies the TestCases to be executed in parallel,
...@@ -1267,31 +1362,48 @@ def _run_cases_in_parallel(cases, fanout=1, runner=None): ...@@ -1267,31 +1362,48 @@ def _run_cases_in_parallel(cases, fanout=1, runner=None):
runner process. runner process.
""" """
if runner is None:
runner = unittest.TextTestRunner(verbosity=2, failfast=True) multi = logging.getLogger("multiprocess")
handler = logging.StreamHandler()
multi.addHandler(handler)
# testq: The master process enqueues TestCase objects into this queue, if VERBOSE:
# test runner processes pick them up for execution, in parallel. multi.setLevel(logging.DEBUG)
testq = Queue() else:
multi.setLevel(logging.INFO)
testq = []
worker_folder = []
runners = [] runners = []
for i in xrange(0,fanout):
testq.append(Queue())
worker_folder.append(os.path.join(image_folder, 'process'+str(i)))
os.mkdir(worker_folder[i])
for i in xrange(0, fanout): for i in xrange(0, fanout):
kwargs = dict(testq=testq, runner=runner) kwargs = dict(testq=testq[i], worker_folder=worker_folder[i])
runners.append(TestRunnerProcess(kwargs=kwargs)) runners.append(TestRunnerProcess(kwargs=kwargs))
log.info("Spawning %d test runner processes", len(runners)) multi.debug("Spawning %d test runner processes" %len(runners))
for p in runners: for p in runners:
p.start() p.start()
log.debug("Spawned %d test runners, PIDs are %s",
len(runners), [p.pid for p in runners])
# Enqueue test cases # Enqueue test cases
map(testq.put, cases) for i in xrange(0, fanout):
map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners)) map(testq[i].put, cases)
testq[i].put("TEST_RUNNER_TERMINATE")
multi.debug("Spawned %d test runners, PIDs are %s" %
(len(runners), [p.pid for p in runners]))
log.debug("Joining %d processes", len(runners)) multi.debug("Joining %d processes" % len(runners))
for p in runners: for p in runners:
p.join() p.join()
log.debug("Done joining %d processes", len(runners))
multi.debug("Done joining %d processes" % len(runners))
def _spawn_server_test_case(**kwargs): def _spawn_server_test_case(**kwargs):
...@@ -1308,7 +1420,9 @@ def _spawn_server_test_case(**kwargs): ...@@ -1308,7 +1420,9 @@ def _spawn_server_test_case(**kwargs):
# Make sure the class can be pickled, by listing it among # Make sure the class can be pickled, by listing it among
# the attributes of __main__. A PicklingError is raised otherwise. # the attributes of __main__. A PicklingError is raised otherwise.
setattr(__main__, name, cls)
thismodule = sys.modules[__name__]
setattr(thismodule, name, cls)
return cls return cls
...@@ -1320,7 +1434,9 @@ def _spawn_network_test_case(**kwargs): ...@@ -1320,7 +1434,9 @@ def _spawn_network_test_case(**kwargs):
# Make sure the class can be pickled, by listing it among # Make sure the class can be pickled, by listing it among
# the attributes of __main__. A PicklingError is raised otherwise. # the attributes of __main__. A PicklingError is raised otherwise.
setattr(__main__, name, cls)
thismodule = sys.modules[__name__]
setattr(thismodule, name, cls)
return cls return cls
...@@ -1347,15 +1463,32 @@ def cleanup_servers(delete_stale=False): ...@@ -1347,15 +1463,32 @@ def cleanup_servers(delete_stale=False):
print >> sys.stderr, "Use --delete-stale to delete them." print >> sys.stderr, "Use --delete-stale to delete them."
def cleanup_networks(delete_stale=False): def cleanup_networks(action_timeout, query_interval,delete_stale=False):
def isSnfTest(s):
if s.find(SNF_TEST_PREFIX) == -1:
return False
else:
return True
c = CycladesClient(API, TOKEN) c = CycladesClient(API, TOKEN)