Commit 15ce514d authored by Dimitris Aragiorgis's avatar Dimitris Aragiorgis Committed by Christos Stavrakakis

Add snf-deploy tree

Signed-off-by: default avatarDimitris Aragiorgis <dimara@grnet.gr>
parent 254d3991
*.pyc
lala.tmp
kamaki-fab
images/*
*.pid
*.monitor
packages
*.swp
include README Changelog
include distribute_setup.py
#!/bin/bash
usage(){
echo "
Usage: $0: [options]
-h, --help Prints this help message
--debian [branch] Local debian branch to use (default debian)
--upstream [branch] Local upstream branch to use (default master)
--remote [repo] Remote repo to use (default origin)
--packages [dir] Where to store the created packages (default ~/packages)
--validate Fetch remote repo branches and
check if local are up-to-date (default false)
--push Whether to push upstream (default false)
"
exit 1
}
parse_git_branch()
{
git branch 2> /dev/null | grep '^*' | sed 's/^*\ //g'
}
die()
{
echo -e $* 1>&2
echo Aborting.
exit 1
}
cleanup()
{
trap - EXIT
echo -n Cleaning up...
if [ ${#CLEANUP[*]} -gt 0 ]; then
LAST_ELEMENT=$((${#CLEANUP[*]}-1))
REVERSE_INDEXES=$(seq ${LAST_ELEMENT} -1 0)
for i in $REVERSE_INDEXES; do
local cmd=${CLEANUP[$i]}
$cmd
done
fi
echo "done"
}
add_cleanup() {
local cmd=""
for arg; do cmd+=$(printf "%q " "$arg"); done
CLEANUP+=("$cmd")
}
add_checkpoint()
{
commit=$(git reflog | head -n1 | cut -f 1 -d " ")
add_cleanup git reset --hard $commit
LASTCHECKPOINT=$commit
}
CLEANUP=( )
TEMP=$(getopt -o h --long help,validate,push,packages:,upstream:,debian:,remote: -n 'autopkg.sh' -- "$@")
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$TEMP"
while true ; do
case "$1" in
-h|--help) usage ;;
--upstream) LOCALUPSTREAM=$2 ; shift 2 ;;
--debian) LOCALDEBIAN=$2 ; shift 2 ;;
--remote) REMOTE=$2 ; shift 2 ;;
--packages) PKGAREA=$2 ; shift 2 ;;
--validate) VALIDATE=true ; shift ;;
--push) PUSH=true ; shift ;;
--) shift ; break ;;
*) echo "Internal error!" ; usage ;;
esac
done
# The root of the git repository, no matter where we're called from
TOPLEVEL="$(git rev-parse --show-toplevel)"
: ${LOCALUPSTREAM:=$(parse_git_branch)}
: ${LOCALDEBIAN:=debian}
: ${REMOTE:=origin}
: ${VALIDATE:=false}
: ${PUSH:=false}
: ${PKGAREA:=~/packages}
: ${BACKUPAREA:=~/backup}
cd "$TOPLEVEL"
# Prerequisites: Test all important directories exist
test -d "$PKGAREA" || die "Package area directory $PKGAREA missing"
test -d "$BACKUPAREA" || die "Backup area directory $BACKUPAREA missing"
# Prerequisite: Test the dialog utility is available
dialog --help &>/dev/null || die "Could not run the 'dialog' utility"
BUILDAREA=$(mktemp -d --tmpdir=/tmp build-area.XXX)
add_cleanup rm -r $BUILDAREA
echo "############################################################################"
echo "Will build packages under $BUILDAREA"
echo "Local upstream branch: $LOCALUPSTREAM"
echo "Local debian branch: $LOCALDEBIAN"
$VALIDATE && echo "Will fetch $REMOTE and check if $LOCALUPSTREAM and $LOCALDEBIAN are up-to-date"
echo "############################################################################"
echo "Press Enter to continue..."
read
MODIFIED=$(git status --short | grep -v "??")
test -z "$MODIFIED" || die "error: Repository is dirty. Commit your local changes:\n $MODIFIED"
set -e
trap cleanup EXIT
add_checkpoint
# Create a temporary debian branch to do everything
TMPDEBIAN=$(mktemp -u debian.XXX)
git branch --track $TMPDEBIAN $LOCALDEBIAN
add_cleanup git branch -D $TMPDEBIAN
git checkout $TMPDEBIAN
add_cleanup git checkout $LOCALUPSTREAM
# Whether we are in snapshot or release mode
snap=false
mrgextra=-m
dchextra=-R
mrgmsg="Merge branch '$LOCALUPSTREAM' into $LOCALDEBIAN"
dialog --yesno "Create Snapshot?" 5 20 && snap=true && dchextra=-S && mrgextra= && mrgmsg=
# merge local branch into tmp branch with a nice commit message,
# so it can be pushed as is to upstream debian
export GIT_MERGE_AUTOEDIT=no
git merge $mrgextra ${mrgextra:+"$mrgmsg"} $LOCALUPSTREAM
# auto edit Debian changelog depending on Snapshot or Release mode
export EDITOR=/usr/bin/vim
git-dch --debian-branch=$TMPDEBIAN --git-author --ignore-regex=".*" --multimaint-merge --since=HEAD $dchextra
git add debian/changelog
# get version from the changelog
# we add a git tag here, so setup.py sdist works as expected
# FIXME: This is a workaround for the way Synnefo packages determine
# the versions for their Python packages
version=$(IFS="()" ; read x v x < debian/changelog ; echo $v)
if ! $snap; then
git commit -s -a -m "Bump new upstream version"
TAGFILE=$(mktemp -t tag.XXX)
add_cleanup rm $TAGFILE
dialog --inputbox "New Debian Tag: " 5 30 "debian/$version" 2>$TAGFILE
git tag $(<$TAGFILE)
add_cleanup git tag -d $(<$TAGFILE)
fi
add_cleanup git reset --hard HEAD
# Build all packages
git-buildpackage --git-export-dir="$BUILDAREA" \
--git-upstream-branch=$LOCALUPSTREAM \
--git-debian-branch=$TMPDEBIAN \
--git-export=INDEX \
--git-ignore-new -sa
# do some dirty backup
# pkgarea might be needed by auto-deploy tool
rm -f "$PKGAREA"/* || true
cp -v "$BUILDAREA"/* "$PKGAREA"/ || true
cp -v "$BUILDAREA"/* "$BACKUPAREA"/ || true
function check_remote(){
git fetch $1 2>/dev/null || die "Could not fetch $1"
git fetch $1 $2 2>/dev/null|| die "Could not fetch $1/$2"
commits_behind=$(git rev-list $2..$1/$2 | wc -l)
if [ $commits_behind -ne 0 ]; then
die "Your local branch is outdated. Please run:\ngit pull --rebase $1/$2"
fi
}
if $VALIDATE; then
check_remote $REMOTE $LOCALUPSTREAM
check_remote $REMOTE $LOCALDEBIAN
fi
# trap - EXIT
# here we can push the commits to the remote debian branch as they are
echo
echo "#################################################"
echo "## SUCCESS ##"
echo "#################################################"
if $PUSH; then
git push --tags $REMOTE $TMPDEBIAN:$LOCALDEBIAN
git push $REMOTE $LOCALUPSTREAM:$LOCALUPSTREAM
fi
exit 0
[packages]
# whether to use apt-get or local generated package found in packages dir
use_local_packages = True
# url to obtain latest synnefo packages.
# To use them change USE_LOCAL_PACKAGES setting to yes
# To get them run: snf-deploy packages
package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/
[dirs]
# dir to find all template files used to customize setup
# in case you want to add another setting please modify the corresponding file
templates = /var/lib/snf-deploy/files
# dir to store local images (disk0, disk1 of the virtual cluster)
images = /var/lib/snf-deploy/images
# dir to store/find local packages
# dir to locally save packages that will be downloaded from package_url
# put here any locally created packages (useful for development)
packages = /var/lib/snf-deploy/packages
# dir to store pidfiles (dnsmasq, kvm)
run = /var/run/snf-deploy
# dir to store dnsmasq related files
dns = /var/lib/snf-deploy/dnsmasq
# dir to lookup fabfile and ifup script
lib = /usr/lib/snf-deploy
# dir to store executed commands (to enforce sequential execution)
cmd = /var/run/snf-deploy/cmd
[ganeti1]
cluster_nodes = node1
master_node = node1
cluster_netdev = eth0
cluster_name = ganeti1
cluster_ip = 192.168.0.13
vg = autovg
synnefo_public_network_subnet = 10.0.1.0/24
synnefo_public_network_gateway = 10.0.1.1
synnefo_public_network_type = CUSTOM
image_dir = /srv/okeanos
# To add another cluster repeat the above section
# with different header and nodes
# please note that currently is only supported deployment
# with nodes (both ganeti and synnefo) residing in the same subnet/domain
[network]
domain = synnefo.live
subnet = 192.168.0.0/28
gateway = 192.168.0.14
[hostnames]
node1 = auto1
# node2 = auto2
[ips]
node1 = 192.168.0.1
# node2 = 192.168.0.2
# This is used only in case of vcluster
# needed to pass the correct dhcp responces to the virtual nodes
[macs]
node1 = 52:54:00:00:00:01
# node2 = 52:54:00:00:00:02
[info]
# Here we define which nodes from the predefined ones to use
nodes = node1
# login credentials for the nodes
# please note that in case of vcluster these are preconfigured
# and not editable.
# in case of physical nodes all nodes should have the same login account
user = root
password = 12345
public_iface = eth0
vm_public_iface = eth1
vm_private_iface = eth2
# extra disk name inside the nodes
# if defined, snf-deploy will create a VG for ganeti in order to support lvm storage
# if not then only file disk template will be supported
extra_disk = /dev/vdb
[debian]
rabbitmq-server = testing
gunicorn = squeeze-backports
qemu-kvm = squeeze-backports
qemu = squeeze-backports
python-gevent = squeeze-backports
apache2 =
postgresql =
python-psycopg2 =
python-argparse =
nfs-kernel-server = squeeze-backports
nfs-common = squeeze-backports
bind9 =
vlan =
vlan =
lvm2 =
curl =
memcached =
python-memcache =
bridge-utils =
python-progress =
ganeti-instance-debootstrap =
[synnefo]
snf-astakos-app = stable
snf-common = stable
snf-cyclades-app = stable
snf-cyclades-gtools = stable
snf-django-lib = stable
python-astakosclient = stable
python-objpool = stable
snf-branding = stable
snf-webproject = stable
snf-pithos-app = stable
snf-pithos-backend = stable
snf-tools = stable
python-django-south = stable
[ganeti]
snf-ganeti = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze
ganeti-htools = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze
[other]
snf-cloudcms = stable
snf-vncauthproxy = stable
snf-pithos-webclient = stable
snf-image = stable
snf-network = stable
nfdhcpd = stable
kamaki = stable
python-bitarray = stable
nfqueue-bindings-python = stable
[cred]
synnefo_user = synnefo
synnefo_db_passwd = example_passw0rd
synnefo_rapi_passwd = example_rapi_passw0rd
synnefo_rabbitmq_passwd = example_rabbitmq_passw0rd
user_email = user@synnefo.org
user_name = John
user_lastname = Doe
user_passwd = 12345
[roles]
accounts = node1
compute = node1
object-store = node1
cyclades = node1
pithos = node1
cms = node1
db = node1
mq = node1
ns = node1
client = node1
router = node1
[synnefo]
pithos_dir = /srv/pithos
vm_public_bridge = br0
vm_private_bridge = prv0
common_bridge = br0
debian_base_url = https://pithos.okeanos.grnet.gr/public/RDISy7sNVIJ9KIm4JkmbX4
[image]
# url to get the base image. This is a debian base image with preconfigured
# root password and installed rsa/dsa keys. Plus a NetworkManager hook that
# changes the VM's name based on info provided by dhcp response.
# To create it run: snf-deploy image
squeeze_image_url = https://pithos.okeanos.grnet.gr/public/832xv
ubuntu_image_url =
# in order ganeti nodes to support lvm storage (plain disk template) it will
# be needed an extra disk to eventually be able to create a VG. Ganeti requires
# this VG to be at least of 30GB. To this end in order the virtual nodes to have
# this extra disk an image should be created locally. There are three options:
# 1. not create an extra disk (only file storage template will be supported)
# 2. create an image of 30G in image dir (default /var/lib/snf-deploy/images)
# using dd if=/dev/zero of=squeeze.disk1
# 3. create this image in a local VG using lvgreate -L30G squeeze.disk1 lvg
# and create a symbolic link in /var/lib/snf-deploy/images
# Whether to create an extra disk or not
create_extra_disk = False
# lvg is the name of the local VG if any
lvg =
# OS istalled in the virtual cluster
os = squeeze
[cluster]
# the bridge to use for the virtual cluster
# on this bridge we will launch a dnsnmasq and provide
# fqdns needed to the cluster.
# In ordrer cluster nodes to have internet access, host must do NAT.
# iptables -t nat -A POSTROUTING -s 192.0.0.0/28 -j MASQUERADE
# ip addr add 192.0.0.14/28 dev auto_nodes_br
# To create run: snf-deploy cluster
bridge = auto_nodes_br
This diff is collapsed.
This diff is collapsed.
<VirtualHost *:80>
ServerName %HOST%
RewriteEngine On
RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
</VirtualHost>
<IfModule mod_ssl.c>
<VirtualHost _default_:443>
ServerName %HOST%
Alias /static "/usr/share/synnefo/static"
# SetEnv no-gzip
# SetEnv dont-vary
AllowEncodedSlashes On
RequestHeader set X-Forwarded-Protocol "https"
<Proxy * >
Order allow,deny
Allow from all
</Proxy>
SetEnv proxy-sendchunked
SSLProxyEngine off
ProxyErrorOverride off
ProxyPass /static !
ProxyPass / http://localhost:8080/ retry=0
ProxyPassReverse / http://localhost:8080/
# RewriteEngine On
# RewriteRule ^/login(.*) /im/login/redirect\$1 [PT,NE]
SSLEngine on
SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem
SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
</VirtualHost>
</IfModule>
# Backports repository
deb http://backports.debian.org/debian-backports squeeze-backports main contrib non-free
deb http://www.rabbitmq.com/debian/ testing main
deb http://apt.dev.grnet.gr squeeze/
deb http://ftp.de.debian.org/debian wheezy main non-free contrib
//
// Do any local configuration here
//
// Consider adding the 1918 zones here, if they are not used in your
// organization
//include "/etc/bind/zones.rfc1918";
zone "%DOMAIN%" in {
type master;
file "/etc/bind/zones/%DOMAIN%";
};
zone "in-addr.arpa" in {
type master;
file "/etc/bind/rev/synnefo.in-addr.arpa.zone";
};
options {
directory "/var/cache/bind";
// If there is a firewall between you and nameservers you want
// to talk to, you may need to fix the firewall to allow multiple
// ports to talk. See http://www.kb.cert.org/vuls/id/800113
// If your ISP provided one or more IP addresses for stable
// nameservers, you probably want to use them as forwarders.
// Uncomment the following block, and insert the addresses replacing
// the all-0's placeholder.
// forwarders {
// 0.0.0.0;
// };
auth-nxdomain no; # conform to RFC1035
allow-recursion { %NODE_IPS%; };
listen-on-v6 { any; };
};
$TTL 86400
$ORIGIN in-addr.arpa.
@ IN SOA ns.%DOMAIN%. admin.%DOMAIN%. (
2012070900; the Serial Number
172800; the Refresh Rate
7200; the Retry Time
604800; the Expiration Time
3600) ; the Minimum Time
@ IN NS ns.%DOMAIN%.
$TTL 14400
$origin %DOMAIN%.
@ IN SOA ns.%DOMAIN%. admin.%DOMAIN%. (
2012111903; the Serial Number
172800; the Refresh Rate
7200; the Retry Time
604800; the Expiration Time
3600; the Minimum Time
)
@ IN NS ns.%DOMAIN%.
@ IN A %NS_NODE_IP%
ns IN A %NS_NODE_IP%
localhost IN A 127.0.0.1
%DOMAIN%. IN MX 10 %DOMAIN%.
mail IN CNAME %DOMAIN%.
www IN CNAME %DOMAIN%.
# snf-image defaults file
# IMAGE_NAME: Name of the image to use
# Generally you use the name of the image with the version of the OS included.
# Examples include:
# centos-5.4 debian-4.0 fedora-12
# IMAGE_NAME=""
# IMAGE_DIR: directory location for disk images
# IMAGE_DIR="/var/lib/snf-image"
IMAGE_DIR=/srv/okeanos
# IMAGE_DEBUG: turn on debugging output for the scripts
# IMAGE_DEBUG=no
# VERSION_CHECK: Check if host and helper have the
# same version. This is usefull if snf-image-host is
# installed as debian package and not from source.
# VERSION_CHECK="no"
# HELPER_DIR: Directory hosting the helper files
# HELPER_DIR="/var/lib/snf-image/helper/"
# HELPER_CACHE_DIR: Directory hosting the helper cache files
# HELPER_CACHE_DIR="/var/cache/snf-image/helper/"
# HELPER_IMG: Path to the helper VM image
# HELPER_IMG="${HELPER_DIR}/image"
# HELPER_KERNEL: Path to the helper VM kernel
# HELPER_KERNEL="${HELPER_DIR}/kernel"
# HELPER_INITRD: Path to the helper VM initial ramdisk
# HELPER_INITRD="${HELPER_DIR}/initrd"
# HELPER_TIMOUT: Soft and hard timeout limits for helper instance.
# The helper instance will be terminated after a given time if it hasn't exited
# by itself. A TERM signal will be send if the instance is running after
# a HELPER_SOFT_TIMEOUT interval. A KILL signal will be sent, if the instance
# is still running after a HELPER_HARD_TIMEOUT interval since the initial
# signal was sent. The timeout values are integer numbers with an optional
# suffix: `s' for seconds (the default), `m' for minutes, `h' for hours or `d'
# for days.
# HELPER_SOFT_TIMEOUT="20"
# HELPER_HARD_TIMEOUT="5"
HELPER_SOFT_TIMEOUT=100
# HELPER_USER: For security reasons, it is recommended that the helper VM
# runs as an unprivileged user. KVM drops root privileges and runs as
# HELPER_USER imeddiately before starting execution of the helper VM.
# HELPER_USER="nobody"
# MULTISTRAP_CONFIG: Configuration file to be used with multistrap to create
# the rootfs of the helper image.
# MULTISTRAP_CONFIG="/etc/snf-image/multistrap.conf"
# MULTISTRAP_APTPREFDIR: Directory where apt preference files are hosted. Those
# files will be injected to the helper image before multistrap is called.
# MULTISTRAP_APTPREFDIR="/etc/snf-image/apt.pref.d"
# PITHOS_DB: Pithos database in SQLAlchemy format
# PITHOS_DB="sqlite:////var/lib/pithos/backend.db"
PITHOS_DB=postgresql://%SYNNEFO_USER%:%SYNNEFO_DB_PASSWD%@%DB_NODE%:5432/snf_pithos
# PITHOS_DATA: Directory where pithos data are hosted
# PITHOS_DATA="/var/lib/pithos/data"
PITHOS_DATA=%PITHOS_DIR%/data
# PROGRESS_MONITOR: External program that monitors the progress of the image
# deployment. The snf-image monitor messages will be redirected to the standard
# input of this program.
# PROGRESS_MONITOR=""
PROGRESS_MONITOR=snf-progress-monitor
# UNATTEND: This variables overwrites the unattend.xml file used when deploying
# a windows image. snf-image-helper will use its own unattend.xml file if this
# variable is empty. Please unless you really know what you are doing, leave
# this empty.
# UNATTEND=""
# Paths for needed programs. Uncommend and change the variables below if you
# don't want to use the default one.
# LOSETUP="losetup"
# KPARTX="kpartx"
# SFDISK="sfdisk"
# QEMU_IMG="qemu-img"
# INSTALL_MBR="install-mbr"
# TIMELIMIT="timelimit"
# CURL="curl"
CURL="curl -k"
#!/bin/sh
# Filename: /etc/dhcp3/dhclient-exit-hooks.d/hostname
# Purpose: Used by dhclient-script to set the hostname of the system
# to match the DNS information for the host as provided by
# DHCP.
# Depends: dhcp3-client (should be in the base install)
# hostname (for hostname, again, should be in the base)
# bind9-host (for host)
# coreutils (for cut and echo)
#
if [ "$reason" != BOUND ] && [ "$reason" != RENEW ] \
&& [ "$reason" != REBIND ] && [ "$reason" != REBOOT ]
then
return
fi
echo dhclient-exit-hooks.d/hostname: Dynamic IP address = $new_ip_address
hostname=$(host $new_ip_address | sed 's/.$//' | cut -d ' ' -f 5)
echo $hostname > /etc/hostname
hostname $hostname
echo dhclient-exit-hooks.d/hostname: Dynamic Hostname = $hostname
# And that _should_ just about do it...
%PITHOS_DIR% %SUBNET%(rw,sync,no_subtree_check,no_root_squash)
/srv/okeanos %SUBNET%(rw,sync,no_subtree_check,no_root_squash)
%SRV% %SUBNET%(rw,fsid=0,no_subtree_check,sync,no_root_squash)
CONFIG = {
'mode': 'django',
'environment': {
'DJANGO_SETTINGS_MODULE': 'synnefo.settings',
},
'working_dir': '/etc/synnefo',
'user': 'www-data',
'group': 'www-data',
'args': (
'--bind=127.0.0.1:8080',
'--workers=8',
'--worker-class=gevent',
# '--worker-class=sync',
'--log-level=debug',
),
}
## nfdhcpd sample configuration file
## General options
[general]
pidfile = /var/run/nfdhcpd/nfdhcpd.pid
datapath = /var/lib/nfdhcpd # Where the client configuration will be read from
logdir = /var/log/nfdhcpd # Where to write our logs
user = nobody # An unprivileged user to run as