Merge pull request #228 from ioantsaf/service-vm-image-refine

Service vm image refinement

core/fokia/ansible_manager_minimal.py

@@ -26,7 +26,7 @@ class Manager:
         self.ansible_inventory.add_group(ansible_group)
         all_group.add_child_group(ansible_group)
 
-    def run_playbook(self, playbook_file, tags=None):
+    def run_playbook(self, playbook_file, only_tags=None, skip_tags=None):
         """
         Run the playbook_file using created inventory and tags specified
         :return:
@@ -35,7 +35,7 @@ class Manager:
         playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
         runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
         pb = PlayBook(playbook=playbook_file, inventory=self.ansible_inventory, stats=stats,
-                      callbacks=playbook_cb,
-                      runner_callbacks=runner_cb, only_tags=tags)
+                      callbacks=playbook_cb, runner_callbacks=runner_cb,
+                      only_tags=only_tags, skip_tags=skip_tags)
         playbook_result = pb.run()
         return playbook_result

core/fokia/provisioner_base.py

@@ -75,7 +75,7 @@ class ProvisionerBase:
         self.vpn = None
         self.subnet = None
         self.private_key = None
-        self.image_id = 'c6f5adce-21ad-4ce3-8591-acfe7eb73c02'
+        self.image_id = '0ad78ab6-d3cd-42c9-8922-9cf63bbb7539'
 
     """
     FIND RESOURCES

core/tests/test_ansible_manager.py

@@ -36,13 +36,12 @@ def test_playbook_run_minimal_manager():
         # Initialize the ansible manager and run a playbook
         manager = MiniManager("test_host", "test_group", "/path/to/private/keys")
         manager.run_playbook(playbook_file="/path/to/playbook.yml",
-                             tags=["touch"])
+                             only_tags=["touch"], skip_tags=["touch-2"])
 
         # Check playbook was called with the correct arguements
-        pb.assert_called_with(stats='c', only_tags=['touch'],
-                              callbacks='a',
+        pb.assert_called_with(only_tags=["touch"], skip_tags=["touch-2"],
+                              stats='c', callbacks='a', runner_callbacks='b',
                               playbook='/path/to/playbook.yml',
-                              runner_callbacks='b',
                               inventory=mock_ansible.inventory.Inventory.return_value)
 
 
@@ -64,8 +63,7 @@ def test_playbook_run():
         # Check that playbook was called with the correct argumenets
         pb.asset_called_with(stats='c', only_tags=['touch'], skip_tags=["touch-2"], callbacks='a',
                              playbook='/path/to/playbook.yml', runner_callbacks='b',
-                             inventory=manager.inventory
-                             )
+                             inventory=manager.inventory)
 
 
 if __name__ == "__main__":

core/tests/test_provisioner.py

@@ -29,7 +29,7 @@ test_flavors = [{
 
 test_images = [{
     u'created': u'2015-06-26T11:29:59+00:00',
-    u'id': u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02', u'is_snapshot': False,
+    u'id': u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539', u'is_snapshot': False,
     u'links': [
         {u'href': u'https://cyclades.okeanos.grnet.gr/compute/v2.0/images/1232', u'rel': u'self'}, {
             u'href': u'https://cyclades.okeanos.grnet.gr/compute/v2.0/images/234d',
@@ -183,13 +183,13 @@ def test_find_flavor():
         provisioner.astakos.get_projects.return_value = test_projects
         provisioner.cyclades.list_images.return_value = test_images
         provisioner.cyclades.list_flavors.return_value = test_flavors
-        provisioner.image_id = u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02'
+        provisioner.image_id = u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539'
         provisioner.create_vm(vm_name="tost", project_name="lambda.grnet.gr",
                               project_mode="supahpower", image_name="archlinux", net_id="12345",
                               flavor={'id': 3})
         provisioner.cyclades.create_server. \
             assert_called_with(flavor_id=3,
-                               image_id=u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02',
+                               image_id=u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539',
                                name='tost',
                                networks=[{u'uuid': '12345'}],
                                personality=[],
@@ -237,7 +237,7 @@ def test_create_vpn():
 
         # Call the functions to set up a network
         provisioner = Provisioner(None, "lambda")
-        provisioner.image_id = u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02'
+        provisioner.image_id = u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539'
 
         provisioner.network_client.create_network.return_value = test_ip
         provisioner.reserve_ip('6ff62e8e-0ce9-41f7-ad99-13a18ecada5f')
@@ -326,7 +326,7 @@ def test_create_cluster():
 
         # Check that the astakos cyclades call gets the correct parameters
         assert provisioner.cyclades.create_server.call_args_list == [
-            call(name='test_master_name', image_id=u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02',
+            call(name='test_master_name', image_id=u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539',
                  flavor_id=3, project_id=u'6ff62e8e-0ce9-41f7-ad99-13a18ecada5f',
                  networks=[{u'fixed_ip': 'test_ip', u'uuid': 'test_ip'}, {u'uuid': 'test_uuid'}],
                  personality=[{
@@ -340,7 +340,7 @@ def test_create_cluster():
                      'owner': u'root', 'path': u'/root/.ssh/id_rsa', 'group': u'root',
                      'mode': 384, 'contents': 'dGVzdF9wcml2YXRlX2tleQ=='
                  }]),
-            call(name=u'lambda-node1', image_id=u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02',
+            call(name=u'lambda-node1', image_id=u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539',
                  flavor_id=3, project_id=u'6ff62e8e-0ce9-41f7-ad99-13a18ecada5f',
                  networks=[{u'fixed_ip': 'test_ip', u'uuid': 'test_ip'}, {u'uuid': 'test_uuid'}],
                  personality=[{
@@ -348,7 +348,7 @@ def test_create_cluster():
                      'mode': 384,
                      'contents': 'dGVzdF9leHRyYV9wdWJfa2V5CnRlc3RfcHVibGljX2tleSByb290'
                  }]),
-            call(name=u'lambda-node2', image_id=u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02',
+            call(name=u'lambda-node2', image_id=u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539',
                  flavor_id=3, project_id=u'6ff62e8e-0ce9-41f7-ad99-13a18ecada5f',
                  networks=[{u'fixed_ip': 'test_ip', u'uuid': 'test_ip'}, {u'uuid': 'test_uuid'}],
                  personality=[{

core/tests/test_vm_manager.py

@@ -164,7 +164,7 @@ def test_create_single_vm():
             assert_called_with(**{
                 'name': 'test_name',
                 'image_id':
-                    u'c6f5adce-21ad-4ce3-8591-acfe7eb73c02',
+                    u'0ad78ab6-d3cd-42c9-8922-9cf63bbb7539',
                 'flavor_id': 3,
                 'project_id': '234f23f42234f',
                 'networks': [

webapp/ansible/roles/common/tasks/main.yml

@@ -12,5 +12,8 @@
   - name: Install python-pip.
     apt: name=python-pip force=yes
 
+  - name: Install curl
+    apt: name=curl force=yes
+
   - name: Create celery user.
     user: name=celery shell=/bin/bash home=/home/celery

webapp/ansible/roles/service-vm/tasks/main.yml

@@ -44,6 +44,8 @@
 
   - name: Create user fixture
     template: src=users.yaml.j2 dest={{ repository_download_path}}/okeanos-LoD/webapp/backend/fixtures/users.yaml
+    tags: create-user
 
   - name: Load user from fixtures
     django_manage: command=loaddata app_path={{ repository_download_path }}/okeanos-LoD/webapp/ fixtures=users.yaml
+    tags: create-user

webapp/image/README.md

+# Service VM image creation
+
+## Description
+
+The scripts contained in the webapp/image directory are responsible for assisting an
+administrator in creating a new Service VM image.
+
+
+### ansible-vm-init
+
+The run-once init script responsible for  the initial configuration of the Service VM, after it has
+been created using the image. It runs ansible on localhost, using local connection, and including
+ the tags necessary for the initial configurations. Then, after a successful run, the init script
+ disables and deletes itself.
+
+### index.html
+
+The default apache page. It contains the AdminLTE template, and is used as a waiting page for the
+ user, while his Service VM initializes. The page auto-refreshes every 20 seconds, so that when
+ the Service VM has been configured, the Lambda webpage appears.
+
+### setup_default_site.sh
+
+Script responsible for setting up the default apache page. It copies the index.html, along with
+the necessary js/css/image files, taken from the ember project directories.
+
+### create_image.sh
+
+Script responsible for creating a Service VM image. It must be run on a service VM. It sets up
+the default site, enables the ansible-vm-init init script, then runs snf-image-creator, to bundle
+ the host.
+
+## Usage
+
+1. Create a new Service VM, by issuing:
+`webapp/manager/service_vm_manager.py --action image_creation`.
+2. After the service_vm_manager finishes successfully, ssh to the created VM.
+3. Navigate to the image folder on the Service VM: `cd /var/www/okeanos-LoD/webapp/image`.
+4. Run `./create_image.sh`.
+5. snf-image-creator will run, to bundle the host. Insert the required image properties, and
+~okeanos credentials, to register your new image on the ~okeanos cloud. Recommended image
+name/description: Lambda Service VM YYYYMMDD. (eg. Lambda Service VM 20151219)

webapp/image/create_image.sh

+#!/bin/bash
+
+# Setup default site
+/var/www/okeanos-LoD/webapp/image/setup_default_site.sh &&
+
+# Copy and enable run-once init script
+cp /var/www/okeanos-LoD/webapp/image/ansible-vm-init /etc/init.d/ &&
+update-rc.d ansible-vm-init start 3 &&
+
+# Run snf-image-creator
+snf-image-creator /
+

webapp/image/index.html

+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <!--<meta http-equiv="Refresh" content="20; url=/">-->
+    <title>λambda Project</title>
+    <!-- Tell the browser to be responsive to screen width -->
+    <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
+    <!-- Bootstrap 3.3.5 -->
+    <link rel="stylesheet" href="files/bootstrap/css/bootstrap.min.css">
+    <!-- Font Awesome -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css">
+    <!-- Theme style -->
+    <link rel="stylesheet" href="files/css/AdminLTE.min.css">
+    <!-- AdminLTE Skins. Choose a skin from the css/skins
+         folder instead of downloading all of them to reduce the load. -->
+    <link rel="stylesheet" href="files/css/skin-blue.css">
+</head>
+
+<body class="hold-transition skin-blue sidebar-mini">
+<div class="wrapper">
+
+    <header class="main-header">
+        <!-- Logo -->
+        <a href="/" class="logo">
+            <!-- mini logo for sidebar mini 50x50 pixels -->
+            <span class="logo-mini"><b>λ</b></span>
+            <!-- logo for regular state and mobile devices -->
+          <span class="logo-lg"> <img src="files/img/okeanos-ico.png" alt="Okeanos"
+                                      style="margin-bottom:5px;margin-right: 3px;"><b>λambda</b>
+          </span>
+        </a>
+        <!-- Header Navbar: style can be found in header.less -->
+        <nav class="navbar navbar-static-top" role="navigation">
+            <!-- Sidebar toggle button-->
+            <a href="/" class="sidebar-toggle" data-toggle="offcanvas" role="button">
+                <span class="sr-only">Toggle navigation</span>
+            </a>
+        </nav>
+    </header>
+
+
+    <!-- Content Wrapper. Contains page content -->
+    <div class="content-wrapper">
+        <!-- Content Header (Page header) -->
+        <section class="content-header">
+            <h1>
+                Dashboard
+                <small>Login</small>
+            </h1>
+            <h1>
+                Initializing
+                <small><i class="fa fa-cog fa-spin"></i></small>
+            </h1>
+        </section>
+
+        <!-- Main content -->
+        <section class="content">
+            <div class="lockscreen-wrapper">
+                <div class="lockscreen-logo">
+                    <span class="logo-lg"> <img src="files/img/okeanos-ico.png" alt="Okeanos"
+                                                style="margin-bottom:5px;margin-right: 3px;"><b>λambda</b></span>
+                </div>
+                <!-- User name -->
+
+                <!-- START LOCK SCREEN ITEM -->
+                <div class="lockscreen-item">
+                    <!-- lockscreen image -->
+                    <div class="lockscreen-image">
+                        <img src="files/img/default_blogger.png" alt="User Image">
+                    </div>
+                    <!-- /.lockscreen-image -->
+
+                    <!-- lockscreen credentials (contains the form) -->
+                    <form class="lockscreen-credentials">
+                        <div class="input-group">
+                            ~okeanos token
+                            <div class="input-group-btn">
+                                <button class="btn"><i class="fa fa-arrow-right text-muted"></i></button>
+                            </div>
+                        </div>
+                    </form><!-- /.lockscreen credentials -->
+
+                </div><!-- /.lockscreen-item -->
+                <div class="help-block text-center">
+                    Enter your ~okeanos token to retrieve your session
+                </div>
+            </div>
+        </section><!-- /.content -->
+    </div><!-- /.content-wrapper -->
+
+
+    <footer class="main-footer">
+        <div class="row">
+            <div class="col-md-12">
+
+                <a href="http://europa.eu/" target="_blank" title="European Union"><img
+                        src="files/img/european_union.png" alt="European Union" width="130"></a>
+                <a href="http://www.digitalplan.gov.gr/portal/" target="_blank" title="Digital Greece"><img
+                        src="files/img/digitalgreece.png" alt="Digital Greece" width="160"></a>
+                <a href="http://www.espa.gr/en/Pages/Default.aspx" target="_blank" title="NSRF"><img
+                        src="files/img/NSRF.png" alt="NSRF" width="70"></a>
+
+            </div>
+        </div>
+        <div class="row">
+            <div class="col-md-12">
+                The project is co-financed by Greece and the European Union.
+            </div>
+        </div>
+        <div class="pull-right ">
+            <a href="http://okeanos.grnet.gr/" target="_blank" title="Okeanos Project"><img
+                    src="files/img/okeanos_logo_powered_by.svg" alt="Okeanos" width="130"></a>
+        </div>
+        <strong>Copyright &copy; 2015 <a href="http://www.grnet.gr" target="_blank">GRNET S.A.</a>.</strong> All rights
+        reserved.
+    </footer>
+
+
+</div><!-- ./wrapper -->
+
+<!-- jQuery 2.1.4 -->
+<script src="files/js/plugins/jQuery/jQuery-2.1.4.min.js"></script>
+<!-- jQuery UI 1.11.4 -->
+<script src="files/js/plugins/jQueryUI/jquery-ui.min.js"></script>
+<!-- Bootstrap 3.3.5 -->
+<script src="files/bootstrap/js/bootstrap.min.js"></script>
+<!-- AdminLTE App -->
+<script src="files/js/app.min.js"></script>
+
+<script src="files/js/jquery.plainoverlay.min.js"></script>
+<script>
+    $(function () {
+        $('.content-wrapper').plainOverlay('show');
+    })
+</script>
+
+<style>
+    .jQuery-plainOverlay-progress {
+        border-color: #3c8dbc;
+    }
+</style>
+
+<!-- Reload after initialization -->
+<script>
+    $(document).ready(function () {
+        setInterval(function () {
+            $.ajax({
+                url: '/',
+                type: 'GET',
+                error: function () {
+                    // Error is thrown when https ember site comes up, due to cross-origin request
+                    location.href = '/';
+                }
+            });
+        }, 6000);
+    });
+</script>
+
+</body>
+</html>

webapp/image/setup_default_site.sh

+#!/bin/bash
+
+# Copy image files
+cp -r /var/www/okeanos-LoD/webapp/frontend/public/assets/img/ /var/www/html/files
+
+# Copy js and css files
+mkdir -p /var/www/html/files/js
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/dist/js/app.min.js /var/www/html/files/js/
+mkdir -p /var/www/html/files/css
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/dist/css/AdminLTE.min.css /var/www/html/files/css
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/dist/css/skins/skin-blue.css /var/www/html/files/css
+mkdir -p /var/www/html/files/js/plugins/jQuery
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/plugins/jQuery/jQuery-2.1.4.min.js /var/www/html/files/js/plugins/jQuery/
+mkdir -p /var/www/html/files/js/plugins/jQueryUI
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/plugins/jQueryUI/jquery-ui.min.js /var/www/html/files/js/plugins/jQueryUI/
+mkdir -p /var/www/html/files/bootstrap/js
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/bootstrap/js/bootstrap.min.js /var/www/html/files/bootstrap/js/
+mkdir -p /var/www/html/files/bootstrap/css
+cp /var/www/okeanos-LoD/webapp/frontend/bower_components/AdminLTE/bootstrap/css/bootstrap.min.css /var/www/html/files/bootstrap/css/
+
+# Copy default html
+cp /var/www/okeanos-LoD/webapp/image/index.html /var/www/html
+
+# Fetch jquery-plainoverlay js
+wget --quiet https://raw.githubusercontent.com/anseki/jquery-plainoverlay/master/jquery.plainoverlay.min.js -P /var/www/html/files/js
+

webapp/manager/service_vm_manager.py

@@ -37,7 +37,8 @@ class ServiceVMManager(object):
     def service_vm_create(self, vm_name='Service VM',
                           vcpus=4, ram=4096, disk=40,
                           project_name='lambda.grnet.gr',
-                          private_key_path=None, public_key_path=None):
+                          private_key_path=None, public_key_path=None,
+                          only_tags=None, skip_tags=None):
         """
         Creates the service vm and installs the relevant s/w.
         :return: ansible result
@@ -54,7 +55,8 @@ class ServiceVMManager(object):
         self._patch_auth_token_ansible()
         ansible_manager = Manager(hostname, group, private_key_path)
         ansible_result = ansible_manager.run_playbook(
-            playbook_file=os.path.join(ansible_path, 'playbooks', 'setup.yml'))
+            playbook_file=os.path.join(ansible_path, 'playbooks', 'setup.yml'),
+            only_tags=only_tags, skip_tags=skip_tags)
 
         self._clean_up_token_ansible_patch()
         return ansible_result
@@ -122,7 +124,7 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description='Lambda service VM provisioning')
     parser.add_argument('--action', type=str, dest='action', required=True,
-                        choices=['create', 'start', 'stop', 'destroy'])
+                        choices=['create', 'start', 'stop', 'destroy', 'image_creation'])
     parser.add_argument('--auth_token', type=str, dest='auth_token', required=False)
     parser.add_argument('--vm_id', type=int, dest='vm_id')
     parser.add_argument('--vm_name', type=str, dest='vm_name', required=False,
@@ -145,6 +147,13 @@ if __name__ == "__main__":
                              project_name=args.project_name,
                              private_key_path=args.private_key_path,
                              public_key_path=args.public_key_path)
+    elif args.action == 'image_creation':
+        sm.service_vm_create(vm_name=args.vm_name,
+                             vcpus=args.vcpus, ram=args.ram, disk=args.disk,
+                             project_name=args.project_name,
+                             private_key_path=args.private_key_path,
+                             public_key_path=args.public_key_path,
+                             skip_tags=['image-configure', 'create-user'])
     elif args.vm_id is None:
         raise ValueError("VM id must be specified")
     else: