Merge branch 'feature-pithos-backend-storage' of https://github.com/cnanakos/snf-image into develop

docs/architecture.rst

@@ -106,8 +106,12 @@ newly created block device. The following back-ends are supported:
  * **Pithos backend**:
    *snf-image* contains a special command-line tool (*pithcat*) for retrieving
    images residing on a Pithos installation. To set up snf-image's Pithos
-   backend the user needs to setup the ``PITHOS_DATA`` and ``PITHOS_DB``
-   variables inside ``/etc/default/snf-image`` accordingly.
+   backend the user needs to setup the ``PITHOS_BACKEND_STORAGE`` variable
+   inside ``/etc/default/snf-image``.
+   Possible values are ``nfs`` and ``rados``. If ``nfs`` is used the user needs
+   to setup ``PITHOS_DATA`` variable, and when ``rados`` is used the user needs
+   to setup ``PITHOS_RADOS_POOL_MAPS`` and ``PITHOS_RADOS_POOL_BLOCKS``
+   accordingly.
 
  * **Null backend**:
    If the null backend is selected, no image copying is performed. This

docs/configuration.rst

@@ -64,9 +64,25 @@ some external programs in ``/etc/default/snf-image``:
   # PITHOS_DB: Pithos database in SQLAlchemy format
   # PITHOS_DB="sqlite://///var/lib/pithos/backend.db"
 
+  # PITHOS_BACKEND_STORAGE: Select Pithos backend storage. Possible values are
+  # 'nfs' and 'rados'. According to the value you select, you need to set the
+  # corresponding variables that follow.
+  # If you select 'nfs' that's 'PITHOS_DATA'. If you select 'rados' then you
+  # need to set all the "*_RADOS_*" ones.
+  # PITHOS_BACKEND_STORAGE="nfs"
+
   # PITHOS_DATA: Directory where Pithos data are hosted
   # PITHOS_DATA="//var/lib/pithos/data"
 
+  # PITHOS_RADOS_CEPH_CONF: RADOS configuration file
+  # PITHOS_RADOS_CEPH_CONF="@sysconfdir@/ceph/ceph.conf"
+
+  # PITHOS_RADOS_POOL_MAPS: RADOS pool for storing Pithos maps
+  # PITHOS_RADOS_POOL_MAPS="maps"
+
+  # PITHOS_RADOS_POOL_BLOCKS: RADOS pool for storing Pithos blocks
+  # PITHOS_RADOS_POOL_BLOCKS="blocks"
+
   # PROGRESS_MONITOR: External program that monitors the progress of image
   # deployment. Monitoring messages will be redirected to the standard input of
   # this program.

docs/interface.rst

@@ -66,10 +66,12 @@ to be used. If no prefix is used, it defaults to the local back-end:
  * **Pithos backend**:
    If the **img_id** is prefixed with ``pithos://`` or ``pithosmap://`` the
    image is considered to reside on a Pithos deployment. For ``pithosmap://``
-   images, the user needs to have set a valid value for the
-   ``PITHOS_DATA`` variable in snf-image's configuration file
-   (``/etc/default/snf-image`` by default). For ``pithos://`` images, in
-   addition to ``PITHOS_DATA``, the user needs to have set a valid value for the
+   images, the user needs to have set a valid value for the ``PITHOS_DATA``
+   variable in snf-image's configuration file (``/etc/default/snf-image`` by
+   default) if the storage backend is ``nfs`` or ``PITHOS_RADOS_POOL_MAPS`` and
+   ``PITHOS_RADOS_POOL_BLOCKS`` if the storage backend is ``rados``.
+   For ``pithos://`` images, in addition to ``PITHOS_DATA`` or
+   ``PITHOS_RADOS_POOL_*``, the user needs to have set a valid value for the
    ``PITHOS_DB`` variable, too.
 
   | For example, if we want to deploy using a full Pithos URI:

snf-image-host/common.sh.in

@@ -454,6 +454,10 @@ fi
 : ${HELPER_MEMORY:="512"}
 : ${PITHOS_DB:="sqlite:////@localstatedir@/lib/pithos/backend.db"}
 : ${PITHOS_DATA:="@localstatedir@/lib/pithos/data/"}
+: ${PITHOS_BACKEND_STORAGE:="nfs"}
+: ${PITHOS_RADOS_CEPH_CONF:="@sysconfdir@/ceph/ceph.conf"}
+: ${PITHOS_RADOS_POOL_MAPS:="maps"}
+: ${PITHOS_RADOS_POOL_BLOCKS:="blocks"}
 : ${PROGRESS_MONITOR:="@PROGRESS_MONITOR@"}
 : ${UNATTEND:="@UNATTEND@"}
 : ${XEN_SCRIPTS_DIR="@sysconfdir@/xen/scripts"}

snf-image-host/create

@@ -111,6 +111,10 @@ case $BACKEND_TYPE in
         # environmental variable.
         export PITHCAT_INPUT_DB="$PITHOS_DB"
         export PITHCAT_INPUT_DATA="$PITHOS_DATA"
+        export PITHCAT_BACKEND_STORAGE="$PITHOS_BACKEND_STORAGE"
+        export PITHCAT_RADOS_CEPH_CONF="$PITHOS_RADOS_CEPH_CONF"
+        export PITHCAT_RADOS_POOL_MAPS="$PITHOS_RADOS_POOL_MAPS"
+        export PITHCAT_RADOS_POOL_BLOCKS="$PITHOS_RADOS_POOL_BLOCKS"
         cmd_args="$(printf "%q" "${IMAGE_NAME}")"
         image_cmd="./pithcat $cmd_args"
         image_size=$(./pithcat -s  $cmd_args)

snf-image-host/defaults.in

@@ -50,13 +50,29 @@
 # MULTISTRAP_APTPREFDIR="@MULTISTRAP_APTPREFDIR@"
 
 # XEN_SCRIPTS_DIR: Directory where the Xen scripts are stored
-# XEN_SCRIPTS_DIR=="@sysconfdir@/xen/scripts"
+# XEN_SCRIPTS_DIR="@sysconfdir@/xen/scripts"
 
 # PITHOS_DB: Pithos database in SQLAlchemy format
 # PITHOS_DB="sqlite:////@localstatedir@/lib/pithos/backend.db"
 
+# PITHOS_BACKEND_STORAGE: Select Pithos backend storage. Possible values are
+# 'nfs' and 'rados'. According to the value you select, you need to set the
+# corresponding variables that follow.
+# If you select 'nfs' that's 'PITHOS_DATA'. If you select 'rados' then you need
+# to set all the "*_RADOS_*" ones.
+# PITHOS_BACKEND_STORAGE="nfs"
+
 # PITHOS_DATA: Directory where Pithos data are hosted
 # PITHOS_DATA="/@localstatedir@/lib/pithos/data"
+#
+# PITHOS_RADOS_CEPH_CONF: RADOS configuration file
+# PITHOS_RADOS_CEPH_CONF="@sysconfdir@/ceph/ceph.conf"
+
+# PITHOS_RADOS_POOL_MAPS: RADOS pool for storing Pithos maps
+# PITHOS_RADOS_POOL_MAPS="maps"
+
+# PITHOS_RADOS_POOL_BLOCKS: RADOS pool for storing Pithos blocks
+# PITHOS_RADOS_POOL_BLOCKS="blocks"
 
 # PROGRESS_MONITOR: External program that monitors the progress of the image
 # deployment. The snf-image monitor messages will be redirected to the standard

snf-image-host/pithcat

@@ -41,6 +41,14 @@ except ImportError:
 parser = OptionParser(usage='%prog [options] <URL>')
 parser.add_option('--data', dest='data', metavar='DIR',
                   help='path to the directory where data are stored')
+parser.add_option('--backend', dest='backend', metavar='BACKEND',
+                  help='Pithos backend storage type')
+parser.add_option('--rados-conf', dest='rconf', metavar='RCONF',
+                  help='RADOS configuration file to use')
+parser.add_option('--rados-maps', dest='rmaps', metavar='RMAPS',
+                  help='RADOS pool which Pithos maps reside')
+parser.add_option('--rados-blocks', dest='rblocks', metavar='RBLOCKS',
+                  help='RADOS pool which Pithos blocks reside')
 parser.add_option('-s', action='store_true', dest='size', default=False,
                   help='print file size and exit')
 group = OptionGroup(
@@ -133,11 +141,22 @@ def main():
         exit(1)
 
     db_uri = environ['PITHCAT_INPUT_DB'] if not options.db else options.db
-
+    backend_storage = environ['PITHCAT_BACKEND_STORAGE'] if not \
+        options.backend else options.backend
+    rados_ceph_conf = environ['PITHCAT_RADOS_CEPH_CONF'] if not \
+        options.rconf else options.rconf
+    rados_maps = environ['PITHCAT_RADOS_POOL_MAPS'] if not options.rmaps else \
+        options.rmaps
+    rados_blocks = environ['PITHCAT_RADOS_POOL_BLOCKS'] if not \
+        options.rblocks else options.rblocks
+
+    block_params = {'mappool': rados_maps, 'blockpool': rados_blocks}
     backend = ModularBackend(None,
                              db_uri if type(url) is LocationURL else None,
                              None,
-                             data_path)
+                             data_path, block_params=block_params,
+                             backend_storage=backend_storage,
+                             rados_ceph_conf=rados_ceph_conf)
 
     if options.size:
         print_size(backend, url)