Commit a5d6bb05 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis
Browse files

Merge pull request #53 from dimara/feature-deploy-pythonize

Change conf files to proper ConfigParser ones.
Introduce separate config and context modules.
Refactor components so that one can use:
ctx = context.Context()
NS = components.NS(ctx)
NS.setup()
parents 1e6f3ee0 d0dc448b
[packages]
[DEFAULT]
# whether to use apt-get or local generated package found in packages dir
use_local_packages = True
# url to obtain latest synnefo packages.
# To use them change USE_LOCAL_PACKAGES setting to yes
# To get them run: snf-deploy packages
package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/
[dirs]
# dir to store snf-deploy status
state = /var/lib/snf-deploy
state_dir = /var/lib/snf-deploy
# dir to find all template files used to customize setup
# in case you want to add another setting please modify the corresponding file
templates = /var/lib/snf-deploy/files
template_dir = /var/lib/snf-deploy/files
# dir to store local images (disk0, disk1 of the virtual cluster)
images = /var/lib/snf-deploy/images
image_dir = /var/lib/snf-deploy/images
# dir to store/find local packages
# dir to locally save packages that will be downloaded from package_url
# put here any locally created packages (useful for development)
packages = /var/lib/snf-deploy/packages
package_dir = /var/lib/snf-deploy/packages
# dir to store pidfiles (dnsmasq, kvm)
run = /var/run/snf-deploy
run_dir = /var/run/snf-deploy
# dir to store dnsmasq related files
dns = /var/lib/snf-deploy/dnsmasq
dns_dir = /var/lib/snf-deploy/dnsmasq
# dir to lookup fabfile and ifup script
lib = /usr/lib/snf-deploy
# dir to store executed commands (to enforce sequential execution)
cmd = /var/run/snf-deploy/cmd
lib_dir = /usr/lib/snf-deploy
# dir to be used by Django for file-based mail backend
mail_dir = /var/tmp/synnefo-mails
[keys]
# whether to create new keys
keygen = False
# whether to inject ssh keys found in templates/root/.ssh in nodes
key_inject = True
[options]
# Deploy Synnefo, specially tuned for testing. This option improves the speed
# of some operations, but is not safe for all enviroments. (e.g. disable
# fsync of postgresql)
......
[ganeti1]
cluster_nodes =
master_node = node1
cluster_netdev = eth0
cluster_name = ganeti1
cluster_ip = 192.168.0.13
[DEFAULT]
vg = ganeti
vg = autovg
synnefo_public_network_subnet = 10.2.1.0/24
synnefo_public_network_gateway = 10.2.1.1
synnefo_public_network_type = CUSTOM
[ganeti1]
name = ganeti1
domain = synnefo.live
ip = 10.0.2.1
netdev = eth0
# whether to add synnefo related packages
synnefo = True
image_dir = /srv/okeanos
[ganeti2]
name = ganeti2
domain = synnefo.live
ip = 10.0.2.2
netdev = eth0
# whether to add synnefo related packages
synnefo =
# To add another cluster repeat the above section
# with different header and nodes
[ganeti3]
name = ganeti3
domain = synnefo.live
ip = 10.0.2.3
netdev = eth0
# whether to add synnefo related packages
synnefo =
# please note that currently is only supported deployment
# with nodes (both ganeti and synnefo) residing in the same subnet/domain
[network]
# In this section we define configuration setting common to all nodes
[DEFAULT]
# Currently both ganeti and synnefo must reside in the same domain
# Instances will reside in the .vm.<domain> subdomain
domain = synnefo.live
[os]
node1 = wheezy
# node2 = wheezy
# Each node should define:
[hostnames]
node1 = auto1
# node2 = auto2
[ips]
node1 = 192.168.0.1
# node2 = 192.168.0.2
# The node's desired hostname. It will be set
hostname =
# The node's primary IP
ip =
# This is used only in case of vcluster
# needed to pass the correct dhcp responces to the virtual nodes
[macs]
node1 = 52:54:00:00:00:01
# node2 = 52:54:00:00:00:02
[passwords]
node1 = 12345
# node2 = 67890
[info]
# Here we define which nodes from the predefined ones to use
# comma separated node names e.g. node1,node2
nodes = node1
# login credentials for the nodes
# please note that in case of vcluster these are preconfigured
# and not editable.
# in case of physical nodes all nodes should have the same login account
mac =
# The node's OS (debian, ubuntu, etc)
# Currently tested only under debian (wheezy)
os = debian
# The node's administrator account (with root priviledges)
user = root
# The root's password
password = 12345
# The interface with internet access
public_iface = eth0
# The interface for the instances' public traffic
vm_public_iface = eth1
# The interface for the instances' private traffic
vm_private_iface = eth2
# extra disk name inside the nodes
# if defined, snf-deploy will create a VG for ganeti in order to support lvm storage
# if not then only file disk template will be supported
extra_disk = /dev/vdb
# The extra disk for the Ganeti VG needed for plain and drbd disk templates
extra_disk =
[node1]
name = auto1
ip = 192.0.2.1
[node2]
name = auto2
ip = 192.0.2.2
[node3]
name = auto3
ip = 192.0.2.3
[node4]
name = auto4
ip = 192.0.2.4
[node5]
name = auto5
ip = 192.0.2.5
[debian]
python-nfqueue = 0.4+physindev-1~wheezy
python-scapy = 2.2.0+rfc6355-1
blktap-utils = 2.0.90next~21~3a70c2f~1452b09-1
[ubuntu]
[DEFAULT]
ns = node1
client = node1
router = node1
nfs = node1
db = node1
mq = node1
astakos = node1
compute = node1
cyclades = node1
pithos = node1
cms = node1
stats = node1
dev = node1
[synnefo]
clusters =
ganeti1
# ganeti2
[qa]
clusters =
# ganeti1
# ganeti2
ganeti3
[ganeti1]
master = node1
vmc =
node1
[ganeti2]
master = node3
vmc =
node3
[ganeti3]
master = node4
vmc =
node4
node5
[cred]
[DEFAULT]
# various credentials
synnefo_user = synnefo
synnefo_db_passwd = example_passw0rd
synnefo_rapi_passwd = example_rapi_passw0rd
......@@ -10,34 +11,24 @@ user_lastname = Doe
user_passwd = 12345
oa2_secret = 12345
[roles]
accounts = node1
compute = node1
object-store = node1
cyclades = node1
pithos = node1
cms = node1
db = node1
mq = node1
ns = node1
client = node1
router = node1
stats = node1
nfs = node1
[synnefo]
pithos_dir = /srv/pithos
image_dir = /srv/images
archip_dir = /srv/archip
flavor_cpu = 1,2,4,8
flavor_ram = 128,256,512,1024,2048,4096,8192
flavor_disk = 2,5,10,20,40,60,80,100
flavor_storage = file,ext_archipelago
# url to download debian wheezy image
debian_base_url = http://cdn.synnefo.org/debian_base-7.0-x86_64.diskdump
# archipelago segment
segment_size = 512
# options related to synnefo networks
vm_public_bridge = br0
vm_private_bridge = prv0
common_bridge = br0
debian_base_url = http://cdn.synnefo.org/debian_base-7.0-x86_64.diskdump
segment_size = 2048
synnefo_public_network_subnet = 10.2.1.0/24
synnefo_public_network_gateway = 10.2.1.1
synnefo_public_network_type = CUSTOM
[image]
[DEFAULT]
# url to get the base image. This is a debian base image with preconfigured
# root password and installed rsa/dsa keys. Plus a NetworkManager hook that
# changes the VM's name based on info provided by dhcp response.
......@@ -24,8 +24,6 @@ lvg =
# OS istalled in the virtual cluster
os = squeeze
[cluster]
# the bridge to use for the virtual cluster
# on this bridge we will launch a dnsnmasq and provide
# fqdns needed to the cluster.
......@@ -35,6 +33,5 @@ os = squeeze
# To create run: snf-deploy cluster
bridge = auto_nodes_br
[network]
subnet = 192.168.0.0/28
gateway = 192.168.0.14
[debian]
rabbitmq-server =
gunicorn =
qemu-kvm =
qemu =
python-gevent =
apache2 =
postgresql =
python-psycopg2 =
python-argparse =
nfs-kernel-server =
nfs-common =
bind9 =
vlan =
vlan =
lvm2 =
curl =
memcached =
python-memcache =
bridge-utils =
python-progress =
ganeti-instance-debootstrap =
python-django-south =
python-django =
drbd8-utils =
collectd =
dnsutils =
python-svipc =
[synnefo]
snf-astakos-app = wheezy
snf-common = wheezy
snf-cyclades-app = wheezy
snf-cyclades-gtools = wheezy
snf-django-lib = wheezy
python-astakosclient = wheezy
snf-branding = wheezy
snf-webproject = wheezy
snf-pithos-app = wheezy
snf-pithos-backend = wheezy
snf-tools = wheezy
snf-stats-app = wheezy
[ganeti]
snf-ganeti = wheezy
ganeti-htools = wheezy
ganeti-haskell = wheezy
ganeti2 = wheezy
[other]
snf-cloudcms = wheezy
snf-vncauthproxy = wheezy
snf-pithos-webclient = wheezy
snf-image = wheezy
snf-network = wheezy
python-objpool = wheezy
nfdhcpd = wheezy
kamaki =
python-bitarray = wheezy
python-nfqueue = 0.4+physindev-1~wheezy
[archip]
librados2 =
archipelago = experimental
archipelago-dbg = experimental
archipelago-modules-dkms = experimental
archipelago-modules-source = experimental
archipelago-rados = experimental
archipelago-rados-dbg = experimental
libxseg0 = experimental
libxseg0-dbg = experimental
python-archipelago = experimental
python-xseg = experimental
archipelago-ganeti = experimental
......@@ -999,7 +999,7 @@ class SynnefoCI(object):
self.logger.debug("Run snf-deploy")
cmd = """
snf-deploy --disable-colors --autoconf all
snf-deploy --disable-colors --autoconf synnefo
"""
_run(cmd, True)
......
......@@ -62,7 +62,7 @@ To install the whole Synnefo stack run:
.. code-block:: console
# snf-deploy all --autoconf
# snf-deploy synnefo --autoconf
This might take a while depending on the physical host you are running on, since
it will download everything that is necessary, install and configure the whole
......@@ -89,7 +89,7 @@ to access.
Then open a browser and point to:
`https://accounts.synnefo.live/astakos/ui/login`
`https://astakos.synnefo.live/astakos/ui/login`
Local access
------------
......@@ -97,7 +97,7 @@ Local access
If you want to access the installation from the same machine it runs on, just
open a browser and point to:
`https://accounts.synnefo.live/astakos/ui/login`
`https://astakos.synnefo.live/astakos/ui/login`
The default <domain> is set to ``synnefo.live``. A local BIND is already
set up by `snf-deploy` to serve all FQDNs.
......
......@@ -44,7 +44,7 @@ to. The Synnefo roles are described in detail :ref:`here
Note that multiple roles can co-exist in the same node
(virtual or physical).
Currently, `snf-deploy` defines the following roles:
Currently, `snf-deploy` defines the following roles under each setup:
* ns: bind server (DNS)
* db: postgresql server (database)
......@@ -55,7 +55,11 @@ Currently, `snf-deploy` defines the following roles:
* cyclades: compute service
* cms: cms service
* stats: stats service
* ganeti: ganeti node
* clusters: the ganeti clusters
For each cluster we have
* vmc: VM container node
* master: master node
......@@ -90,9 +94,13 @@ The previous roles are combinations of the following software components:
* Network: synnefo networking scripts
* GTools: synnefo tools for ganeti
* GanetiCollectd: collectd config for ganeti nodes
* PithosBackend: the pithos backend
* Archip: The archipelago core
* ArchipGaneti: The tools needed by ganeti for archipelago
Each component defines the following things:
* commands to execute on other components before setup
* commands to check prereqs
* commands to prepare installation
* list of packages to install
......@@ -100,17 +108,32 @@ Each component defines the following things:
* restart/reload commands
* initialization commands
* test commands
* commands to execute on other components after setup
All a components needs is the node info that it gets installed to and the
All a components needs is the context that it gets installed to and the
snf-deploy configuration environment (available after parsing conf files).
The context is basically the target node, role, cluster (if any) and
setup.
.. _conf:
Configuration (a)
=================
All configuration of `snf-deploy` happens by editting the following files under
``/etc/snf-deploy``:
All configuration of `snf-deploy` happens by editting the following simple
ConfigParser files under ``/etc/snf-deploy``.
``setups.conf``
---------------
This file includes all coarse grain info for our available setups. Each
section refers to a generic setup (synnefo, qa, etc) or a specific
ganeti cluster (ganeti1, ganeti2, etc.) Each section includes the
corresponding role mappings. For example if the nameserver should be
installed in node1, the NFS on node2, etc. Each generic setup has also
the cluster meta-role. For example synnefo section can have clusters
ganeti1, ganeti2. Each of them has its own vmcs and master roles (which
map to nodes found in nodes.conf).
``nodes.conf``
--------------
......@@ -293,7 +316,7 @@ To install the Synnefo stack on the existing cluster run:
.. code-block:: console
snf-deploy all -vvv
snf-deploy synnefo -vvv
This might take a while.
......@@ -301,7 +324,7 @@ If this finishes without errors, check for successful installation by visiting
from your local machine (make sure you have already setup your local
``resolv.conf`` to point at the cluster's DNS):
| https://accounts.synnefo.live/astakos/ui/
| https://astakos.synnefo.live/astakos/ui/
and login with:
......@@ -326,11 +349,12 @@ Adding another Ganeti Backend
From version 0.12, Synnefo supports multiple Ganeti backends.
`snf-deploy` defines them in ``ganeti.conf``.
After adding another section in ``ganeti.conf``, run:
After adding another section in ``ganeti.conf`` with synnefo setting
set True, run:
.. code-block:: console
snf-deploy backend --cluster-name ganeti2 -vvv
snf-deploy ganeti --cluster ganeti2 -vvv
snf-deploy for Ganeti
......@@ -341,7 +365,7 @@ by issuing:
.. code-block:: console
snf-deploy ganeti --cluster-name ganeti3 -vvv
snf-deploy ganeti --cluster ganeti3 -vvv
snf-deploy as a DevTool
......@@ -355,26 +379,19 @@ option to ``True``. Then run:
.. code-block:: console
snf-deploy run <action1> [<action2>..]
to execute predefined actions or:
.. code-block:: console
snf-deploy run setup --node nodeX \
--role ROLE | --component COMPONENT --method METHOD
snf-deploy run setup --setup SETUP --node nodeX \
--role ROLE --cluster CLUSTER
to setup a synnefo role on a target node or run a specific component's method.
to setup a specific role on a target node of a specific cluster and setup.
For instance, to add another node to an existing ganeti backend run:
.. code-block:: console
snf-deploy run setup --node5 --role ganeti --cluster-name ganeti3
snf-deploy run setup --node node5 --role vmc --cluster ganeti3
`snf-deploy` keeps track of installed components per node in
``/etc/snf-deploy/status.conf``. If a deployment command fails, the developer
can make the required fix and then re-run the same command; `snf-deploy` will
not re-install components that have been already setup and their status
is ``ok``.
``/var/lib/snf-deploy/snf_deploy_status``. If a deployment command
fails, the developer can make the required fix and then re-run the same
command; `snf-deploy` will not re-install components that have been
already setup and their status is ``ok``.
[packages]
[DEFAULT]
# whether to use apt-get or local generated package found in packages dir
use_local_packages = True
# url to obtain latest synnefo packages.
# To use them change USE_LOCAL_PACKAGES setting to yes
# To get them run: snf-deploy packages
package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/
[dirs]
# dir to store snf-deploy status
state = /var/lib/snf-deploy
state_dir = /var/lib/snf-deploy
# dir to find all template files used to customize setup
# in case you want to add another setting please modify the corresponding file
templates = /var/lib/snf-deploy/files
template_dir = /var/lib/snf-deploy/files
# dir to store local images (disk0, disk1 of the virtual cluster)
images = /var/lib/snf-deploy/images
image_dir = /var/lib/snf-deploy/images
# dir to store/find local packages
# dir to locally save packages that will be downloaded from package_url
# put here any locally created packages (useful for development)
packages = /var/lib/snf-deploy/packages
package_dir = /var/lib/snf-deploy/packages
# dir to store pidfiles (dnsmasq, kvm)
run = /var/run/snf-deploy
run_dir = /var/run/snf-deploy
# dir to store dnsmasq related files
dns = /var/lib/snf-deploy/dnsmasq
dns_dir = /var/lib/snf-deploy/dnsmasq
# dir to lookup fabfile and ifup script
lib = /usr/lib/snf-deploy
# dir to store executed commands (to enforce sequential execution)
cmd = /var/run/snf-deploy/cmd
lib_dir = /usr/lib/snf-deploy
# dir to be used by Django for file-based mail backend
mail_dir = /var/tmp/synnefo-mails
[keys]
# whether to create new keys
keygen = False
# whether to inject ssh keys found in templates/root/.ssh in nodes
key_inject = True
[options]
# Deploy Synnefo, specially tuned for testing. This option improves the speed
# of some operations, but is not safe for all enviroments. (e.g. disable
# fsync of postgresql)
......
[ganeti1]
cluster_nodes =
master_node = node1
cluster_netdev = eth0
cluster_name = ganeti1
cluster_ip = 192.168.0.13
[DEFAULT]
vg = ganeti
vg = autovg
synnefo_public_network_subnet = 10.2.1.0/24
synnefo_public_network_gateway = 10.2.1.1
synnefo_public_network_type = CUSTOM
[ganeti1]
name = ganeti1
domain = synnefo.live
ip = 10.0.2.1