Commit cc40aa60 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis
Browse files

snf-ci: Bulid pithos-web-client package

Synnefo depends on pithos-web-client. Up until now
we were downloading this package from apt.dev.grnet.gr.
But when we make a change in Synnefo that also affects
pithos-web-client, we have to upload a new version of
pithos-web-client.

This patch instructs snf-ci to clone and build
the latest pithos-web-client.
parent d1768170
......@@ -5,21 +5,31 @@ build_timeout = 240
apt_repo =
deb http://apt.dev.grnet.gr squeeze/
deb http://packages.x2go.org/debian squeeze main
# Synnefo git repo.
# If not set, snf-ci will copy and use the local repo.
synnefo_repo = https://code.grnet.gr/git/synnefo
# Git branch to test (specify sha1 or branch name). If not set, the
# branch/sha will result from the current repository.
synnefo_branch =
# pithos-web-client git repo
pithos_webclient_repo = https://code.grnet.gr/git/pithos-web-client
# Git branch to use for pithos-web-client
# If not set, snf-ci will decide which one to use
pithos_webclient_branch =
# Defines the schema that snf-deploy will use
schema = one_node_squeeze
# Local dir to save builded packages
pkgs_dir = /tmp/synnefo_pkgs
# If True patch the pydist.py module (see Debian bug #657665)
patch_pydist = True
# Configuration of git (on remote server)
git_config_name = Buildbot
git_config_mail = synnefo@builder.dev.grnet.gr
# Network address from which we allow access to server.
# If not set, access to server is not restricted.
accept_ssh_from =
......
......@@ -5,21 +5,31 @@ build_timeout = 240
apt_repo =
deb http://apt.dev.grnet.gr wheezy/
deb http://packages.x2go.org/debian wheezy main
# Synnefo git repo.
# If not set, snf-ci will copy and use the local repo.
synnefo_repo = https://code.grnet.gr/git/synnefo
# Git branch to test (specify sha1 or branch name). If not set, the
# branch/sha will result from the current repository.
synnefo_branch =
# pithos-web-client git repo
pithos_webclient_repo = https://code.grnet.gr/git/pithos-web-client
# Git branch to use for pithos-web-client
# If not set, snf-ci will decide which one to use
pithos_webclient_branch =
# Defines the schema that snf-deploy will use
schema = one_node_wheezy
# Local dir to save builded packages
pkgs_dir = /tmp/synnefo_pkgs
# If True patch the pydist.py module (see Debian bug #657665)
patch_pydist = False
# Configuration of git (on remote server)
git_config_name = Buildbot
git_config_mail = synnefo@builder.dev.grnet.gr
# Network address from which we allow access to server.
# If not set, access to server is not restricted.
accept_ssh_from =
......
......@@ -146,7 +146,7 @@ def main(): # Too many branches. pylint: disable-msg=R0912
ssh_keys=options.ssh_keys)
synnefo_ci.clone_repo(local_repo=options.local_repo)
if getattr(options, BUILD_SYNNEFO_CMD, False):
synnefo_ci.build_synnefo()
synnefo_ci.build_packages()
if options.fetch_packages:
dest = os.path.abspath(options.fetch_packages)
synnefo_ci.fetch_packages(dest=dest)
......
#!/usr/bin/env python
# !/usr/bin/env python
"""
Synnefo ci utils module
......@@ -155,7 +155,9 @@ class SynnefoCI(object):
self.temp_config.optionxform = str
self.temp_config.read(self.temp_config_file)
self.build_id = build_id
self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
if build_id is not None:
self.logger.info("Will use \"%s\" as build id" %
_green(self.build_id))
# Set kamaki cloud
if cloud is not None:
......@@ -225,7 +227,7 @@ class SynnefoCI(object):
self.logger.error(
"Waiting for server to become %s timed out" % new_status)
self.destroy_server(False)
sys.exit(-1)
sys.exit(1)
elif server['status'] == current_status:
# Sleep for #n secs and continue
timeout = timeout - sleep_time
......@@ -234,7 +236,7 @@ class SynnefoCI(object):
self.logger.error(
"Server failed with status %s" % server['status'])
self.destroy_server(False)
sys.exit(-1)
sys.exit(1)
@_check_kamaki
def destroy_server(self, wait=True):
......@@ -562,7 +564,7 @@ class SynnefoCI(object):
self.logger.debug("Remote file has sha256 hash %s" % hash2)
if hash1 != hash2:
self.logger.error("Hashes differ.. aborting")
sys.exit(-1)
sys.exit(1)
@_check_fabric
def clone_repo(self, local_repo=False):
......@@ -577,6 +579,14 @@ class SynnefoCI(object):
self.config.get('Global', 'git_config_mail'))
_run(cmd, False)
# Clone synnefo_repo
synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo)
# Clone pithos-web-client
self.clone_pithos_webclient_repo(synnefo_branch)
@_check_fabric
def clone_synnefo_repo(self, local_repo=False):
"""Clone Synnefo repo to remote server"""
# Find synnefo_repo and synnefo_branch to use
synnefo_repo = self.config.get('Global', 'synnefo_repo')
synnefo_branch = self.config.get("Global", "synnefo_branch")
......@@ -590,9 +600,9 @@ class SynnefoCI(object):
subprocess.Popen(
["git", "rev-parse", "--short", "HEAD"],
stdout=subprocess.PIPE).communicate()[0].strip()
self.logger.info("Will use branch \"%s\"" % _green(synnefo_branch))
self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
if local_repo or synnefo_branch == "":
if local_repo or synnefo_repo == "":
# Use local_repo
self.logger.debug("Push local repo to server")
# Firstly create the remote repo
......@@ -609,7 +619,7 @@ class SynnefoCI(object):
-q "$@"' > {4}
chmod u+x {4}
export GIT_SSH="{4}"
echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
rm -f {4}
""".format(fabric.env.password,
fabric.env.user,
......@@ -619,46 +629,111 @@ class SynnefoCI(object):
os.system(cmd)
else:
# Clone Synnefo from remote repo
# Currently clonning synnefo can fail unexpectedly
cloned = False
for i in range(10):
self.logger.debug("Clone synnefo from %s" % synnefo_repo)
try:
_run("git clone %s synnefo" % synnefo_repo, False)
cloned = True
break
except BaseException:
self.logger.warning(
"Clonning synnefo failed.. retrying %s" % i)
if not cloned:
self.logger.error("Can not clone Synnefo repo.")
sys.exit(-1)
self.logger.debug("Clone synnefo from %s" % synnefo_repo)
self._git_clone(synnefo_repo)
# Checkout the desired synnefo_branch
self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
cmd = """
cd synnefo
for branch in `git branch -a | grep remotes | \
grep -v HEAD | grep -v master`; do
for branch in `git branch -a | grep remotes | grep -v HEAD`; do
git branch --track ${branch##*/} $branch
done
git checkout %s
""" % (synnefo_branch)
_run(cmd, False)
return synnefo_branch
@_check_fabric
def build_synnefo(self):
"""Build Synnefo packages"""
self.logger.info("Build Synnefo packages..")
self.logger.debug("Install development packages")
def clone_pithos_webclient_repo(self, synnefo_branch):
"""Clone Pithos WebClient repo to remote server"""
# Find pithos_webclient_repo and pithos_webclient_branch to use
pithos_webclient_repo = \
self.config.get('Global', 'pithos_webclient_repo')
pithos_webclient_branch = \
self.config.get('Global', 'pithos_webclient_branch')
# Clone pithos-webclient from remote repo
self.logger.debug("Clone pithos-webclient from %s" %
pithos_webclient_repo)
self._git_clone(pithos_webclient_repo)
# Track all pithos-webclient branches
cmd = """
cd pithos-web-client
for branch in `git branch -a | grep remotes | grep -v HEAD`; do
git branch --track ${branch##*/} $branch > /dev/null 2>&1
done
git branch
"""
webclient_branches = _run(cmd, False)
webclient_branches = webclient_branches.split()
# If we have pithos_webclient_branch in config file use this one
# else try to use the same branch as synnefo_branch
# else use an appropriate one.
if pithos_webclient_branch == "":
if synnefo_branch in webclient_branches:
pithos_webclient_branch = synnefo_branch
else:
# If synnefo_branch starts with one of
# 'master', 'hotfix'; use the master branch
if synnefo_branch.startswith('master') or \
synnefo_branch.startswith('hotfix'):
pithos_webclient_branch = "master"
# If synnefo_branch starts with one of
# 'develop', 'feature'; use the develop branch
elif synnefo_branch.startswith('develop') or \
synnefo_branch.startswith('feature'):
pithos_webclient_branch = "develop"
else:
self.logger.waring(
"Cannot determine which pithos-web-client branch to "
"use based on \"%s\" synnefo branch. "
"Will use develop." % synnefo_branch)
pithos_webclient_branch = "develop"
# Checkout branch
self.logger.debug("Checkout \"%s\" branch" %
_green(pithos_webclient_branch))
cmd = """
cd pithos-web-client
git checkout {0}
""".format(pithos_webclient_branch)
_run(cmd, False)
def _git_clone(self, repo):
"""Clone repo to remote server
Currently clonning from code.grnet.gr can fail unexpectedly.
So retry!!
"""
cloned = False
for i in range(1, 11):
try:
_run("git clone %s" % repo, False)
cloned = True
break
except BaseException:
self.logger.warning("Clonning failed.. retrying %s/10" % i)
if not cloned:
self.logger.error("Can not clone repo.")
sys.exit(1)
@_check_fabric
def build_packages(self):
"""Build packages needed by Synnefo software"""
self.logger.info("Install development packages")
cmd = """
apt-get update
apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
python-dev python-all python-pip --yes --force-yes
python-dev python-all python-pip ant --yes --force-yes
pip install -U devflow
"""
_run(cmd, False)
# Patch pydist bug
if self.config.get('Global', 'patch_pydist') == "True":
self.logger.debug("Patch pydist.py module")
cmd = r"""
......@@ -668,7 +743,15 @@ class SynnefoCI(object):
_run(cmd, False)
# Build synnefo packages
self.logger.debug("Build synnefo packages")
self.build_synnefo()
# Build pithos-web-client packages
self.build_pithos_webclient()
@_check_fabric
def build_synnefo(self):
"""Build Synnefo packages"""
self.logger.info("Build Synnefo packages..")
cmd = """
devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
"""
......@@ -692,6 +775,24 @@ class SynnefoCI(object):
"""
_run(cmd, False)
@_check_fabric
def build_pithos_webclient(self):
"""Build pithos-web-client packages"""
self.logger.info("Build pithos-web-client packages..")
cmd = """
devflow-autopkg snapshot -b ~/webclient_build-area --no-sign
"""
with fabric.cd("pithos-web-client"):
_run(cmd, True)
# Setup pithos-web-client packages for snf-deploy
self.logger.debug("Copy webclient debs to snf-deploy packages dir")
cmd = """
cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/
"""
_run(cmd, False)
@_check_fabric
def build_documentation(self):
"""Build Synnefo documentation"""
......@@ -824,6 +925,7 @@ class SynnefoCI(object):
if not os.path.exists(dest):
os.makedirs(dest)
self.fetch_compressed("synnefo_build-area", dest)
self.fetch_compressed("webclient_build-area", dest)
self.logger.info("Downloaded debian packages to %s" %
_green(dest))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment