Commit fcb74522 authored by Christos Stavrakakis's avatar Christos Stavrakakis
Browse files

Merge branch 'feature-ci-deploy3' into develop

parents b37c2112 6a654b7f
#!/usr/bin/env python
"""
Build Synnefo packages for debian
"""
from utils import SynnefoCI
def autopkg_debian():
"""Build synnefo packages for debian"""
synnefo_ci = SynnefoCI()
synnefo_ci.build_synnefo()
if __name__ == "__main__":
autopkg_debian()
#!/usr/bin/env python
"""
Deploy Synnefo using snf-deploy
"""
from utils import SynnefoCI
def deploy_synnefo():
"""Deploy Synnefo using snf-deploy"""
synnefo_ci = SynnefoCI()
synnefo_ci.deploy_synnefo()
if __name__ == "__main__":
deploy_synnefo()
#!/usr/bin/env python
"""
Download Synnefo packages
"""
from utils import SynnefoCI
def fetch_packages():
"""Download Synnefo packages"""
synnefo_ci = SynnefoCI()
synnefo_ci.fetch_packages()
if __name__ == "__main__":
fetch_packages()
......@@ -5,8 +5,9 @@ build_timeout = 240
apt_repo = http://apt.dev.grnet.gr squeeze/
# Synnefo git repo
synnefo_repo = https://code.grnet.gr/git/synnefo
# Git branch to test (specify sha1 or branch name)
synnefo_branch = HEAD
# Git branch to test (specify sha1 or branch name). If not set, the
# branch/sha will result from the current repository.
synnefo_branch =
# snf-deploy git repo
deploy_repo = https://code.grnet.gr/git/snf-deploy
# Defines the schema that snf-deploy will use
......@@ -20,19 +21,16 @@ git_config_name = Buildbot
git_config_mail = synnefo@builder.dev.grnet.gr
# Url to fetch ssh public keys
public_ssh_keys_url =
# Network address from which we allow access to server
filter_access_network = 195.251.29.0/24,62.217.123.39
# Network address from which we allow access to server. If not set, access
# to server is not restricted.
filter_access_network =
# Config file to save temporary options (eg IPs, passwords etc)
temporary_config = /tmp/ci_temp_conf
[Deployment]
# Credentials
auth_url = https://accounts.okeanos.io/identity/v2.0/
token =
# If we deploy on okeanos.io we have to set this to True
# The server will reside besides a NAT and we have to compute ssh port
deploy_on_io = True
# Choose the 'cloud' to use from .kamakirc
kamaki_cloud =
# Server name to use for our machine
server_name = Synnefo Deployment
# Flavor ID to use
......
#!/usr/bin/env python
"""
Run burnin functional test suite
"""
from utils import SynnefoCI
def run_burnin():
"""Run burnin functional test suite"""
synnefo_ci = SynnefoCI()
synnefo_ci.run_burnin()
if __name__ == "__main__":
run_burnin()
......@@ -35,6 +35,7 @@ snf-webproject = stable
snf-pithos-app = stable
snf-pithos-backend = stable
snf-tools = stable
python-django-south = squeeze-backports
[ganeti]
......
#!/usr/bin/env python
"""
Setup slave server
"""
from utils import SynnefoCI
def setup_slave():
"""Setup slave server"""
synnefo_ci = SynnefoCI(cleanup_config=True)
# Get token from /nfs/token
try:
token = open("/nfs/synnefo_token").read().strip()
synnefo_ci.write_config('token', token, 'Deployment')
except:
pass
# Build slave server
synnefo_ci.create_server()
# Copy synnefo repo to server
synnefo_ci.clone_repo()
if __name__ == "__main__":
setup_slave()
#!/usr/bin/env python
"""
"""
import os
from utils import SynnefoCI
from optparse import OptionParser
CREATE_SERVER_CMD = "create"
BUILD_SYNNEFO_CMD = "build"
BUILD_DOCS_SYNNEFO_CMD = "docs"
DEPLOY_SYNNEFO_CMD = "deploy"
TEST_SYNNEFO_CMD = "test"
RUN_BURNIN_CMD = "burnin"
ALL_CMDS = "all"
AVAILABLE_COMMANDS = [
CREATE_SERVER_CMD,
BUILD_SYNNEFO_CMD,
BUILD_DOCS_SYNNEFO_CMD,
DEPLOY_SYNNEFO_CMD,
TEST_SYNNEFO_CMD,
RUN_BURNIN_CMD,
]
CLEAN_CONFIG_CMDS = [CREATE_SERVER_CMD, ALL_CMDS]
USAGE = """usage: %%prog [options] command
command:
* %s: Create the slave server
* %s: Create debian packages for Synnefo in the created server
* %s: Create documentation for Synnefo in the created server
* %s: Deploy Synnefo in created server
* %s: Run Synnefo unittests
* %s: Run snf-burnin in the deployed Synnefo
* %s: Run all the available commands
""" % tuple([CREATE_SERVER_CMD,
BUILD_SYNNEFO_CMD,
BUILD_DOCS_SYNNEFO_CMD,
DEPLOY_SYNNEFO_CMD,
TEST_SYNNEFO_CMD,
RUN_BURNIN_CMD,
ALL_CMDS])
def main():
parser = OptionParser(usage=USAGE)
parser.add_option("--conf", dest="config_file", default=None,
help="Configuration file for SynnefoCI script"),
parser.add_option("-c", "--cloud", dest="kamaki_cloud", default=None,
help="Use specified cloud, as is in .kamakirc"),
parser.add_option("-f", "--flavor", dest="flavor", default=None,
help="Name of flavor to use for the server.")
parser.add_option("-i", "--image", dest="image", default=None,
help="UUID of image to use for the server.")
parser.add_option("--fetch-packages", dest="fetch_packages",
default=None,
help="Download the debian packages that were created"
" during the '%s' step in this directory" %
BUILD_SYNNEFO_CMD)
parser.add_option("--fetch-docs", dest="fetch_docs",
default=None,
help="Download the documentation that was created"
" during the '%s' step in this directory" %
BUILD_DOCS_SYNNEFO_CMD)
parser.add_option("--schema", dest="schema", default=None,
help="Schema for snf-deploy.")
(options, args) = parser.parse_args()
if len(args) != 1:
msg = "ERROR: Command takes exactly one argument"
parser.print_help()
print
print msg
return
command = args[0]
if command == ALL_CMDS:
for cmd in AVAILABLE_COMMANDS:
setattr(options, cmd, True)
elif command not in AVAILABLE_COMMANDS:
msg = "ERROR: Unknown command: %s" % command
parser.print_help()
print
print msg
return
else:
setattr(options, command, True)
cleanup_config = command in CLEAN_CONFIG_CMDS
synnefo_ci = SynnefoCI(config_file=options.config_file,
cleanup_config=cleanup_config,
cloud=options.kamaki_cloud)
if getattr(options, CREATE_SERVER_CMD, False):
synnefo_ci.create_server(flavor_id=options.flavor,
image_id=options.image)
synnefo_ci.clone_repo()
if getattr(options, BUILD_SYNNEFO_CMD, False):
synnefo_ci.build_synnefo()
if options.fetch_packages:
dest = os.path.abspath(options.fetch_packages)
synnefo_ci.fetch_packages(dest=dest)
if getattr(options, BUILD_DOCS_SYNNEFO_CMD, False):
synnefo_ci.build_documentation()
if options.fetch_docs:
dest = os.path.abspath(options.fetch_docs)
synnefo_ci.fetch_documentation(dest=dest)
if getattr(options, DEPLOY_SYNNEFO_CMD, False):
synnefo_ci.deploy_synnefo(schema=options.schema)
if getattr(options, TEST_SYNNEFO_CMD, False):
synnefo_ci.unit_test()
if getattr(options, RUN_BURNIN_CMD, False):
synnefo_ci.run_burnin()
if __name__ == "__main__":
main()
#!/usr/bin/env python
"""
Run Synnefo unit test suite
"""
from utils import SynnefoCI
def unit_test():
"""Run Synnefo unit test suite"""
synnefo_ci = SynnefoCI()
synnefo_ci.unit_test()
if __name__ == "__main__":
unit_test()
......@@ -9,12 +9,17 @@ import sys
import time
import logging
import fabric.api as fabric
import subprocess
import tempfile
from ConfigParser import ConfigParser, DuplicateSectionError
from kamaki.cli import config as kamaki_config
from kamaki.clients.astakos import AstakosClient
from kamaki.clients.cyclades import CycladesClient
from kamaki.clients.image import ImageClient
DEFAULT_CONFIG_FILE = "new_config"
def _run(cmd, verbose):
"""Run fabric with verbose level"""
......@@ -46,21 +51,21 @@ def _green(msg):
def _check_fabric(fun):
"""Check if fabric env has been set"""
def wrapper(self, *args):
def wrapper(self, *args, **kwargs):
"""wrapper function"""
if not self.fabric_installed:
self.setup_fabric()
return fun(self, *args)
return fun(self, *args, **kwargs)
return wrapper
def _check_kamaki(fun):
"""Check if kamaki has been initialized"""
def wrapper(self, *args):
def wrapper(self, *args, **kwargs):
"""wrapper function"""
if not self.kamaki_installed:
self.setup_kamaki()
return fun(self, *args)
return fun(self, *args, **kwargs)
return wrapper
......@@ -84,7 +89,7 @@ class _MyFormatter(logging.Formatter):
class SynnefoCI(object):
"""SynnefoCI python class"""
def __init__(self, cleanup_config=False):
def __init__(self, config_file=None, cleanup_config=False, cloud=None):
""" Initialize SynnefoCI python class
Setup logger, local_dir, config and kamaki
......@@ -101,10 +106,14 @@ class SynnefoCI(object):
self.repo_dir = os.path.dirname(self.ci_dir)
# Read config file
self.conffile = os.path.join(self.ci_dir, "new_config")
if config_file is None:
config_file = DEFAULT_CONFIG_FILE
if not os.path.isabs(config_file):
config_file = os.path.join(self.ci_dir, config_file)
self.config = ConfigParser()
self.config.optionxform = str
self.config.read(self.conffile)
self.config.read(config_file)
temp_config = self.config.get('Global', 'temporary_config')
if cleanup_config:
try:
......@@ -114,6 +123,16 @@ class SynnefoCI(object):
else:
self.config.read(self.config.get('Global', 'temporary_config'))
# Set kamaki cloud
if cloud is not None:
self.kamaki_cloud = cloud
elif self.config.has_option("Deployment", "kamaki_cloud"):
kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
if kamaki_cloud == "":
self.kamaki_cloud = None
else:
self.kamaki_cloud = None
# Initialize variables
self.fabric_installed = False
self.kamaki_installed = False
......@@ -125,10 +144,16 @@ class SynnefoCI(object):
Setup cyclades_client and image_client
"""
self.logger.info("Setup kamaki client..")
auth_url = self.config.get('Deployment', 'auth_url')
config = kamaki_config.Config()
if self.kamaki_cloud is None:
self.kamaki_cloud = config.get_global("default_cloud")
self.logger.info("Setup kamaki client, using cloud '%s'.." %
self.kamaki_cloud)
auth_url = config.get_cloud(self.kamaki_cloud, "url")
self.logger.debug("Authentication URL is %s" % _green(auth_url))
token = self.config.get('Deployment', 'token')
token = config.get_cloud(self.kamaki_cloud, "token")
#self.logger.debug("Token is %s" % _green(token))
astakos_client = AstakosClient(auth_url, token)
......@@ -179,16 +204,20 @@ class SynnefoCI(object):
self._wait_transition(server_id, "ACTIVE", "DELETED")
@_check_kamaki
def create_server(self):
def create_server(self, image_id=None, flavor_id=None):
"""Create slave server"""
self.logger.info("Create a new server..")
image = self._find_image()
self.logger.debug("Will use image \"%s\"" % _green(image['name']))
self.logger.debug("Image has id %s" % _green(image['id']))
if image_id is None:
image = self._find_image()
self.logger.debug("Will use image \"%s\"" % _green(image['name']))
image_id = image["id"]
self.logger.debug("Image has id %s" % _green(image_id))
if flavor_id is None:
flavor_id = self.config.getint("Deployment", "flavor_id")
server = self.cyclades_client.create_server(
self.config.get('Deployment', 'server_name'),
self.config.getint('Deployment', 'flavor_id'),
image['id'])
flavor_id,
image_id)
server_id = server['id']
self.write_config('server_id', server_id)
self.logger.debug("Server got id %s" % _green(server_id))
......@@ -197,25 +226,25 @@ class SynnefoCI(object):
self.logger.debug("Server's admin user is %s" % _green(server_user))
server_passwd = server['adminPass']
self.write_config('server_passwd', server_passwd)
self.logger.debug(
"Server's admin password is %s" % _green(server_passwd))
server = self._wait_transition(server_id, "BUILD", "ACTIVE")
self._get_server_ip_and_port(server)
self._copy_ssh_keys()
self.setup_fabric()
self.logger.info("Setup firewall")
accept_ssh_from = self.config.get('Global', 'filter_access_network')
self.logger.debug("Block ssh except from %s" % accept_ssh_from)
cmd = """
local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
cut -d':' -f2 | cut -d' ' -f1)
iptables -A INPUT -s localhost -j ACCEPT
iptables -A INPUT -s $local_ip -j ACCEPT
iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
iptables -A INPUT -p tcp --dport 22 -j DROP
""".format(accept_ssh_from)
_run(cmd, False)
if accept_ssh_from != "":
self.logger.debug("Block ssh except from %s" % accept_ssh_from)
cmd = """
local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
cut -d':' -f2 | cut -d' ' -f1)
iptables -A INPUT -s localhost -j ACCEPT
iptables -A INPUT -s $local_ip -j ACCEPT
iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
iptables -A INPUT -p tcp --dport 22 -j DROP
""".format(accept_ssh_from)
_run(cmd, False)
def _find_image(self):
"""Find a suitable image to use
......@@ -237,9 +266,8 @@ class SynnefoCI(object):
def _get_server_ip_and_port(self, server):
"""Compute server's IPv4 and ssh port number"""
self.logger.info("Get server connection details..")
# XXX: check if this IP is from public network
server_ip = server['attachments'][0]['ipv4']
if eval(self.config.get('Deployment', 'deploy_on_io')):
if ".okeanos.io" in self.cyclades_client.base_url:
tmp1 = int(server_ip.split(".")[2])
tmp2 = int(server_ip.split(".")[3])
server_ip = "gate.okeanos.io"
......@@ -251,6 +279,22 @@ class SynnefoCI(object):
self.write_config('server_port', server_port)
self.logger.debug("Server's ssh port is %s" % _green(server_port))
@_check_fabric
def _copy_ssh_keys(self):
if not self.config.has_option("Deployment", "ssh_keys"):
return
authorized_keys = self.config.get("Deployment",
"ssh_keys")
if authorized_keys != "" and os.path.exists(authorized_keys):
keyfile = '/tmp/%s.pub' % fabric.env.user
_run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
fabric.put(authorized_keys, keyfile)
_run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
_run('rm %s' % keyfile, False)
self.logger.debug("Uploaded ssh authorized keys")
else:
self.logger.debug("No ssh keys found")
def write_config(self, option, value, section="Temporary Options"):
"""Write changes back to config file"""
try:
......@@ -268,9 +312,10 @@ class SynnefoCI(object):
fabric.env.user = self.config.get('Temporary Options', 'server_user')
fabric.env.host_string = \
self.config.get('Temporary Options', 'server_ip')
fabric.env.port = self.config.getint('Temporary Options', 'server_port')
fabric.env.password = \
self.config.get('Temporary Options', 'server_passwd')
fabric.env.port = self.config.getint('Temporary Options',
'server_port')
fabric.env.password = self.config.get('Temporary Options',
'server_passwd')
fabric.env.connection_attempts = 10
fabric.env.shell = "/bin/bash -c"
fabric.env.disable_known_hosts = True
......@@ -309,38 +354,43 @@ class SynnefoCI(object):
_run(cmd, False)
synnefo_repo = self.config.get('Global', 'synnefo_repo')
synnefo_branch = self.config.get("Global", "synnefo_branch")
if synnefo_branch == "":
synnefo_branch =\
subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"],
stdout=subprocess.PIPE).communicate()[0].strip()
if synnefo_branch == "HEAD":
synnefo_branch = \
subprocess.Popen(["git", "rev-parse","--short", "HEAD"],
stdout=subprocess.PIPE).communicate()[0].strip()
self.logger.info("Will use branch %s" % synnefo_branch)
# Currently clonning synnefo can fail unexpectedly
for i in range(3):
cloned = False
for i in range(10):
self.logger.debug("Clone synnefo from %s" % synnefo_repo)
try:
_run("git clone %s" % synnefo_repo, False)
_run("git clone %s synnefo" % synnefo_repo, False)
cloned = True
break
except:
self.logger.warning("Clonning synnefo failed.. retrying %s" % i)
synnefo_branch = self.config.get('Global', 'synnefo_branch')
if synnefo_branch == "HEAD":
# Get current branch
synnefo_branch = os.popen("git rev-parse HEAD").read().strip()
self.logger.debug(
"Checkout %s in feature-ci branch" % synnefo_branch)
with fabric.cd("synnefo"):
_run("git checkout -b feature-ci %s" % synnefo_branch, False)
elif synnefo_branch == "origin/master":
pass
elif "origin" in synnefo_branch:
self.logger.debug("Checkout %s branch" % synnefo_branch)
with fabric.cd("synnefo"):
_run("git checkout -t %s" % synnefo_branch, False)
else:
self.logger.debug(
"Checkout %s in feature-ci branch" % synnefo_branch)
with fabric.cd("synnefo"):
_run("git checkout -b feature-ci %s" % synnefo_branch, False)
self.logger.warning("Clonning synnefo failed.. retrying %s"
% i)
cmd ="""
cd synnefo
for branch in `git branch -a | grep remotes | grep -v HEAD | grep -v master`; do
git branch --track ${branch##*/} $branch
done
git checkout %s
""" % (synnefo_branch)
_run(cmd, False)
if not cloned:
self.logger.error("Can not clone Synnefo repo.")
sys.exit(-1)
deploy_repo = self.config.get('Global', 'deploy_repo')
self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
_run("git clone %s" % deploy_repo, False)
_run("git clone --depth 1 %s" % deploy_repo, False)
@_check_fabric
def build_synnefo(self):
......@@ -355,7 +405,7 @@ class SynnefoCI(object):
"""
_run(cmd, False)
if eval(self.config.get('Global', 'patch_pydist')):
if self.config.get('Global', 'patch_pydist') == "True":
self.logger.debug("Patch pydist.py module")
cmd = r"""
sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
......@@ -396,17 +446,39 @@ class SynnefoCI(object):
"""
_run(cmd, False)
@_check_fabric
def build_documentation(self):
self.logger.info("Build Synnefo documentation..")
_run("pip install -U Sphinx", False)
with fabric.cd("synnefo"):
_run("./ci/make_docs.sh synnefo_documentation", False)
def fetch_documentation(self, dest=None):
if dest is None:
dest = "synnefo_documentation"
dest = os.path.abspath(dest)
if not os.path.exists(dest):
os.makedirs(dest)
self.fetch_compressed("synnefo/synnefo_documentation", dest)
self.logger.info("Downloaded documentation to %s" %
_green(dest))
@_check_fabric
def deploy_synnefo(self):
def deploy_synnefo(self, schema=None):
"""Deploy Synnefo using snf-deploy"""
self.logger.info("Deploy Synnefo..")
schema = self.config.get('Global', 'schema')
schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
if schema is None:
schema = self.config.get('Global', 'schema')
self.logger.debug("Will use %s schema" % schema)
schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
raise ValueError("Unknown schema: %s" % schema)
self.logger.debug("Upload schema files to server")
with fabric.quiet():
fabric.put(schema_files, "/etc/snf-deploy/")
fabric.put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
self.logger.debug("Change password in nodes.conf file")
cmd = """
......@@ -467,23 +539,38 @@ class SynnefoCI(object):
_run(cmd, True)