Merge branch 'feature-ci' into develop

Conflicts:
	devflow/autopkg.py
	devflow/versioning.py

COPYRIGHT

-Copyright (C) 2010, 2011, 2012 GRNET S.A. All rights reserved.
+Copyright (C) 2012, 2013 GRNET S.A. All rights reserved.
 
 Redistribution and use in source and binary forms, with or
 without modification, are permitted provided that the following

ci/autopkg_debian.sh

+#!/usr/bin/env sh
+
+BUILD_NUMBER=$1
+BUILDBOT_BUILD_DIR=buildpkg_debian
+
+shift
+
+TEMP_DIR=$(mktemp -d /tmp/devflow_autopkg_XXXXXXX)
+
+# Create the packages
+devflow-autopkg snapshot -b $TEMP_DIR $@
+
+# MOVE the packages
+mkdir -p buildpkg_debian/$BUILD_NUMBER
+mv -n $TEMP_DIR/* $BUILDBOT_BUILD_DIR/$BUILD_NUMBER/

ci/pep8.sh

+#!/usr/bin/env sh
+
+pep8 devflow ci

ci/pylint.sh

+#!/usr/bin/env sh
+
+pylint devflow ci

ci/tests.sh

+#!/usr/bin/env sh
+
+nosetests

devflow.conf

+[ packages ]
+  [[ devflow ]]
+    version_file = "devflow/version.py"

devflow/autopkg.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -37,6 +37,7 @@ import sys
 from optparse import OptionParser
 from collections import namedtuple
 from sh import mktemp, cd, rm, git_dch, python
+from configobj import ConfigObj
 
 from devflow import versioning
 
@@ -116,6 +117,20 @@ def main():
     parser.add_option("-c", "--config-file",
                       dest="config_file",
                       help="Override default configuration file")
+    parser.add_option("--no-sign",
+                      dest="sign",
+                      action="store_false",
+                      default=True,
+                      help="Do not sign the packages")
+    parser.add_option("--key-id",
+                      dest="keyid",
+                      help="Use this keyid for gpg signing")
+    parser.add_option("--dist",
+                      dest="dist",
+                      default="unstable",
+                      help="If running in snapshot mode, automatically set"
+                            " the changelog distribution to this value"
+                            " (default=unstable).")
 
     (options, args) = parser.parse_args()
 
@@ -134,7 +149,7 @@ def main():
         raise ValueError(red("Invalid argument! Mode must be one: %s"
                          % ", ".join(AVAILABLE_MODES)))
 
-    os.environ["GITFLOW_BUILD_MODE"] = mode
+    os.environ["DEVFLOW_BUILD_MODE"] = mode
 
     # Load the repository
     try:
@@ -148,51 +163,38 @@ def main():
         raise RuntimeError(red("Repository %s is dirty." % toplevel))
 
     # Get packages from configuration file
-    config_file = options.config_file or os.path.join(toplevel, "autopkg.conf")
-    packages = get_packages_to_build(config_file)
-    if packages:
-        print_green("Will build the following packages:\n"
-                    "\n".join(packages))
-    else:
-        raise RuntimeError("Configuration file is empty."
-                           " No packages to build.")
-
-    # Clone the repo
-    repo_dir = options.repo_dir
-    if not repo_dir:
-        repo_dir = create_temp_directory("df-repo")
-        print_green("Created temporary directory '%s' for the cloned repo."
-                    % repo_dir)
-
-    repo = original_repo.clone(repo_dir)
-    print_green("Cloned current repository to '%s'." % repo_dir)
+    config_file = options.config_file or os.path.join(toplevel, "devflow.conf")
+    config = ConfigObj(config_file)
+    packages = config['packages'].keys()
+    print_green("Will build the following packages:\n" + "\n".join(packages))
 
-    reflog_hexsha = repo.head.log()[-1].newhexsha
-    print "Latest Reflog entry is %s" % reflog_hexsha
+    # Get current branch name and type and check if it is a valid one
+    branch = original_repo.head.reference.name
+    branch_type = versioning.get_branch_type(branch)
 
-    branch = repo.head.reference.name
-    allowed_branches = ", ".join(x for x in BRANCH_TYPES.keys())
-    if branch.split('-')[0] not in allowed_branches:
+    if branch_type not in BRANCH_TYPES.keys():
+        allowed_branches = ", ".join(BRANCH_TYPES.keys())
         raise ValueError("Malformed branch name '%s', cannot classify as"
                          " one of %s" % (branch, allowed_branches))
 
-    brnorm = versioning.normalize_branch_name(branch)
-    btypestr = versioning.get_branch_type(brnorm)
-
-    # Find the debian branch, and create it if does not exist
-    debian_branch = "debian-" + brnorm
+    # Check that original repo has the correct debian branch
+    debian_branch = "debian-" + branch
     origin_debian = "origin/" + debian_branch
-    if not origin_debian in repo.references:
+    if not debian_branch in original_repo.branches:
         # Get default debian branch
-        try:
-            default_debian = BRANCH_TYPES[btypestr].default_debian_branch
-            origin_debian = "origin/" + default_debian
-        except KeyError:
-            allowed_branches = ", ".join(x for x in BRANCH_TYPES.keys())
-            raise ValueError("Malformed branch name '%s', cannot classify as"
-                             " one of %s" % (btypestr, allowed_branches))
-
-    repo.git.branch("--track", debian_branch, origin_debian)
+        default_debian = BRANCH_TYPES[branch_type].default_debian_branch
+        origin_debian = "origin/" + default_debian
+        if not default_debian in original_repo.branches:
+            original_repo.git.branch(default_debian,
+                                     origin_debian)
+
+    # Clone the repo
+    repo_dir = options.repo_dir or create_temp_directory("df-repo")
+    repo = original_repo.clone(repo_dir)
+    print_green("Cloned repository to '%s'." % repo_dir)
+
+    # Create the debian branch
+    repo.git.branch(debian_branch, origin_debian)
     print_green("Created branch '%s' to track '%s'" % (debian_branch,
                 origin_debian))
 
@@ -202,7 +204,7 @@ def main():
 
     # Merge with starting branch
     repo.git.merge(branch)
-    print_green("Merged branch '%s' into '%s'" % (brnorm, debian_branch))
+    print_green("Merged branch '%s' into '%s'" % (branch, debian_branch))
 
     # Compute python and debian version
     cd(repo_dir)
@@ -211,6 +213,15 @@ def main():
         debian_version_from_python_version(python_version)
     print_green("The new debian version will be: '%s'" % debian_version)
 
+    # Update the version files
+    python("update_version.py", _out=sys.stdout)
+
+    # Tag branch with python version
+    branch_tag = python_version
+    repo.git.tag(branch_tag, branch)
+    upstream_tag = "upstream/" + branch_tag
+    repo.git.tag(upstream_tag, branch)
+
     # Update changelog
     dch = git_dch("--debian-branch=%s" % debian_branch,
                   "--git-author",
@@ -221,90 +232,88 @@ def main():
     print_green("Successfully ran '%s'" % " ".join(dch.cmd))
 
     if mode == "release":
-        # Commit changelog and update tag branches
-        os.system("vim debian/changelog")
-        repo.git.add("debian/changelog")
-        repo.git.commit("-s", "-a", m="Bump new upstream version")
-        python_tag = python_version
-        debian_tag = "debian/" + python_tag
-        repo.git.tag(debian_tag)
-        repo.git.tag(python_tag, brnorm)
+        call("vim debian/changelog")
     else:
         f = open("debian/changelog", 'r+')
         lines = f.readlines()
-        lines[0] = lines[0].replace("UNRELEASED", "unstable")
-        lines[2] = lines[2].replace("UNRELEASED", "Snapshot version")
+        lines[0] = lines[0].replace("UNRELEASED", options.dist)
+        lines[2] = lines[2].replace("UNRELEASED", "Snapshot build")
         f.seek(0)
         f.writelines(lines)
         f.close()
-        repo.git.add("debian/changelog")
-
-    # Update the python version files
-    # TODO: remove this
-    for package in packages:
-        # python setup.py should run in its directory
-        cd(package)
-        package_dir = repo_dir + "/" + package
-        res = python(package_dir + "/setup.py", "sdist", _out=sys.stdout)
-        print res.stdout
-        if package != ".":
-            cd("../")
+
+    # Add changelog to INDEX
+    repo.git.add("debian/changelog")
+    # Commit Changes
+    repo.git.commit("-s", "-a", m="Bump version to %s" % debian_version)
+    # Tag debian branch
+    debian_branch_tag = "debian/" + branch_tag
+    repo.git.tag(debian_branch_tag)
 
     # Add version.py files to repo
-    os.system("grep \"__version_vcs\" -r . -l -I | xargs git add -f")
+    call("grep \"__version_vcs\" -r . -l -I | xargs git add -f")
 
-    # Create debian branches
-    build_dir = options.build_dir
-    if not options.build_dir:
-        build_dir = create_temp_directory("df-build")
-        print_green("Created directory '%s' to store the .deb files." %
-                    build_dir)
+    # Create debian packages
+    build_dir = options.build_dir or create_temp_directory("df-build")
+    print_green("Build directory: '%s'" % build_dir)
 
     cd(repo_dir)
-    os.system("git-buildpackage --git-export-dir=%s --git-upstream-branch=%s"
-              " --git-debian-branch=%s --git-export=INDEX --git-ignore-new -sa"
-              % (build_dir, brnorm, debian_branch))
+    version_files = []
+    for _, pkg_info in config['packages'].items():
+        version_files.append(pkg_info['version_file'])
+    ignore_regexp = "|".join(["^(%s)$" % vf for vf in version_files])
+    build_cmd = "git-buildpackage --git-export-dir=%s"\
+                " --git-upstream-branch=%s --git-debian-branch=%s"\
+                " --git-export=INDEX --git-ignore-new -sa"\
+                " --source-option='\"--extend-diff-ignore=%s\"'"\
+                " --git-upstream-tag=%s"\
+                % (build_dir, branch, debian_branch, ignore_regexp,
+                   upstream_tag)
+    if not options.sign:
+        build_cmd += " -uc -us"
+    elif options.keyid:
+        build_cmd += " -k\"'%s'\"" % options.keyid
+    call(build_cmd)
 
     # Remove cloned repo
     if mode != 'release' and not options.keep_repo:
         print_green("Removing cloned repo '%s'." % repo_dir)
         rm("-r", repo_dir)
-    else:
-        print_green("Repository dir '%s'" % repo_dir)
 
-    print_green("Completed. Version '%s', build area: '%s'"
-                % (debian_version, build_dir))
+    # Print final info
+    info = (("Version", debian_version),
+            ("Upstream branch", branch),
+            ("Upstream tag", branch_tag),
+            ("Debian branch", debian_branch),
+            ("Debian tag", debian_branch_tag),
+            ("Repository directory", repo_dir),
+            ("Packages directory", build_dir))
+    print_green("\n".join(["%s: %s" % (name, val) for name, val in info]))
 
     # Print help message
     if mode == "release":
-        TAG_MSG = "Tagged branch %s with tag %s\n"
-        print_green(TAG_MSG % (brnorm, python_tag))
-        print_green(TAG_MSG % (debian_branch, debian_tag))
+        origin = original_repo.remote().url
+        repo.create_remote("original_origin", origin)
+        print_green("Created remote 'original_origin' for the repository '%s'"
+                    % origin)
 
-        UPDATE_MSG = "To update repository %s, go to %s, and run the"\
-                     " following commands:\n" + "git push origin %s\n" * 3
+        print_green("To update repositories '%s' and '%s' go to '%s' and run:"
+                    % (toplevel, origin, repo_dir))
+        for remote in ['origin', 'original_origin']:
+            print
+            for obj in [debian_branch, branch_tag, debian_branch_tag]:
+                print_green("git push %s %s" % (remote, obj))
 
-        origin_url = repo.remotes['origin'].url
-        remote_url = original_repo.remotes['origin'].url
 
-        print_green(UPDATE_MSG % (origin_url, repo_dir, debian_branch,
-                    debian_tag, python_tag))
-        print_green(UPDATE_MSG % (remote_url, original_repo.working_dir,
-                    debian_branch, debian_tag, python_tag))
+def create_temp_directory(suffix):
+    create_dir_cmd = mktemp("-d", "/tmp/" + suffix + "-XXXXX")
+    return create_dir_cmd.stdout.strip()
 
 
-def get_packages_to_build(config_file):
-    config_file = os.path.abspath(config_file)
-    try:
-        f = open(config_file)
-    except IOError as e:
-        raise IOError("Can not access configuration file %s: %s"
-                      % (config_file, e.strerror))
-
-    lines = [l.strip() for l in f.readlines()]
-    l = [l for l in lines if not l.startswith("#")]
-    f.close()
-    return l
+def call(cmd):
+    rc = os.system(cmd)
+    if rc:
+        raise RuntimeError("Command '%s' failed!" % cmd)
 
 
 def create_temp_directory(suffix):

devflow/versioning.py

 #!/usr/bin/env python
 #
-# Copyright (C) 2010, 2011, 2012 GRNET S.A. All rights reserved.
+# Copyright (C) 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -38,11 +38,11 @@ import os
 import re
 import sys
 import pprint
-import subprocess
 import git
 
 from distutils import log
 from collections import namedtuple
+from configobj import ConfigObj
 
 
 # Branch types:
@@ -83,7 +83,7 @@ def get_commit_id(commit, current_branch):
     elif len(parents) == 2:
         if cur_br_name.startswith("debian-") or cur_br_name == "debian":
             pr1, pr2 = parents
-            return short_id(pr1) + "-" + short_id(pr2)
+            return short_id(pr1) + "_" + short_id(pr2)
         else:
             return short_id(commit)
     else:
@@ -133,15 +133,15 @@ def base_version(vcs_info):
 
 
 def build_mode():
-    """Determine the build mode from the value of $GITFLOW_BUILD_MODE"""
+    """Determine the build mode from the value of $DEVFLOW_BUILD_MODE"""
     try:
-        mode = os.environ["GITFLOW_BUILD_MODE"]
+        mode = os.environ["DEVFLOW_BUILD_MODE"]
         assert mode == "release" or mode == "snapshot"
     except KeyError:
-        raise ValueError("GITFLOW_BUILD_MODE environment variable is not set."
+        raise ValueError("DEVFLOW_BUILD_MODE environment variable is not set."
                          " Set this variable to 'release' or 'snapshot'")
     except AssertionError:
-        raise ValueError("GITFLOW_BUILD_MODE environment variable must be"
+        raise ValueError("DEVFLOW_BUILD_MODE environment variable must be"
                          " 'release' or 'snapshot'")
     return mode
 
@@ -428,24 +428,20 @@ def user_info():
     return "%s@%s" % (getpass.getuser(), socket.getfqdn())
 
 
-def update_version(module, name="version", root="."):
+def update_version():
     """
-    Generate or replace version.py as a submodule of `module`.
-
-    This is a helper to generate/replace a version.py file containing version
-    information as a submodule of passed `module`.
+    This is a helper to generate/replace version files containing version
+    information.
 
     """
 
-    paths = [root] + module.split(".") + ["%s.py" % name]
-    module_filename = os.path.join(*paths)
+    config = ConfigObj("devflow.conf")
 
     v = vcs_info()
     if not v:
         # Return early if not in development environment
-        log.error("Can not compute version outside of a git repository."
-                  " Will not update %s version file" % module_filename)
-        return
+        raise RuntimeError("Can not compute version outside of a git"
+                           " repository.")
     b = base_version(v)
     mode = build_mode()
     version = python_version(b, v, mode)
@@ -457,10 +453,12 @@ __version_user_info__ = "%(user_info)s"
            vcs_info=pprint.PrettyPrinter().pformat(dict(v._asdict())),
            user_info=user_info())
 
-    module_file = file(module_filename, "w+")
-    module_file.write(content)
-    module_file.close()
-    return module_filename
+    for pkg_name, pkg_info in config['packages'].items():
+        version_filename = pkg_info['version_file']
+        log.info("Updating version file '%s'" % version_filename)
+        version_file = file(version_filename, "w+")
+        version_file.write(content)
+        version_file.close()
 
 
 def bump_version_main():

setup.py

-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following
@@ -42,15 +42,6 @@ from fnmatch import fnmatchcase
 from setuptools import setup, find_packages
 
 HERE = os.path.abspath(os.path.normpath(os.path.dirname(__file__)))
-try:
-    from devflow import versioning
-    # use devflow to update the version file
-    versioning.update_version('devflow', 'version', HERE)
-except ImportError:
-    version_fpath = os.path.join(HERE, 'devflow', 'version.py')
-    sys.stdout.write("WARNING: Can not update version because `devflow` is"
-                    " not installed. Please make sure to manually"
-                    " update version file %s" % version_fpath)
 
 from devflow.version import __version__
 
@@ -68,7 +59,7 @@ CLASSIFIERS = []
 
 # Package requirements
 INSTALL_REQUIRES = [
-    'gitpython', 'sh',
+    'gitpython', 'sh', 'configobj'
 ]
 
 # Provided as an attribute, so you can append to these instead
@@ -187,6 +178,7 @@ setup(
      'console_scripts': [
          'devflow-version=devflow.versioning:main',
          'devflow-bump-version=devflow.versioning:bump_version_main',
+         'devflow-update-version=devflow.versioning:update_version',
          'devflow-autopkg=devflow.autopkg:main',
          ],
       },

tests/test_versioning.py

 #!/usr/bin/env python
 #
-# Copyright 2012 GRNET S.A. All rights reserved.
+# Copyright 2012, 2013 GRNET S.A. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or
 # without modification, are permitted provided that the following

update_version.py

+#Copyright (C) 2013 GRNET S.A. All rights reserved.
+#
+#Redistribution and use in source and binary forms, with or
+#without modification, are permitted provided that the following
+#conditions are met:
+#
+#  1. Redistributions of source code must retain the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer.
+#
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials
+#     provided with the distribution.
+#
+#THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+#OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+#CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+#AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+#LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+#ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#POSSIBILITY OF SUCH DAMAGE.
+#
+#The views and conclusions contained in the software and
+#documentation are those of the authors and should not be
+#interpreted as representing official policies, either expressed
+#or implied, of GRNET S.A.
+
+import sys
+try:
+    from devflow import versioning
+except ImportError:
+    raise RuntimeError("devflow is a build dependency")
+
+
+def main():
+    versioning.update_version()
+
+
+if __name__ == "__main__":
+    sys.exit(main())