autopkg.py 13.8 KB
Newer Older
1
# Copyright 2012-2014 GRNET S.A. All rights reserved.
Vangelis Koukis's avatar
Vangelis Koukis committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.

Nikos Skalkotos's avatar
Nikos Skalkotos committed
34
"""Helper script for automatic build of Debian packages."""
Christos Stavrakakis's avatar
Christos Stavrakakis committed
35

Vangelis Koukis's avatar
Vangelis Koukis committed
36
37
import os
import sys
38
39

from git import GitCommandError
Vangelis Koukis's avatar
Vangelis Koukis committed
40
from optparse import OptionParser
Christos Stavrakakis's avatar
Christos Stavrakakis committed
41
from sh import mktemp, cd, rm, git_dch  # pylint: disable=E0611
Vangelis Koukis's avatar
Vangelis Koukis committed
42

43
from devflow import versioning
44
from devflow import utils
45
from devflow import BRANCH_TYPES
Christos Stavrakakis's avatar
Christos Stavrakakis committed
46

Vangelis Koukis's avatar
Vangelis Koukis committed
47
48
49

AVAILABLE_MODES = ["release", "snapshot"]

Nikos Skalkotos's avatar
Nikos Skalkotos committed
50
DESCRIPTION = """Tool for automatic build of Debian packages.
Christos Stavrakakis's avatar
Christos Stavrakakis committed
51

Nikos Skalkotos's avatar
Nikos Skalkotos committed
52
%(prog)s is a helper script for automatic build of Debian packages from
Christos Stavrakakis's avatar
Christos Stavrakakis committed
53
54
55
repositories that follow the `git flow` development model
<http://nvie.com/posts/a-successful-git-branching-model/>.

56
57
This script must run from inside a clean git repository and will perform the
following steps:
Christos Stavrakakis's avatar
Christos Stavrakakis committed
58
59
60
61
62
    * Clone your repository to a temporary directory
    * Merge the current branch with the corresponding debian branch
    * Compute the version of the new package and update the python
      version files
    * Create a new entry in debian/changelog, using `git-dch`
Nikos Skalkotos's avatar
Nikos Skalkotos committed
63
    * Create the Debian packages, using `git-buildpackage`
Christos Stavrakakis's avatar
Christos Stavrakakis committed
64
65
    * Tag the appropriate branches if in `release` mode

Nikos Skalkotos's avatar
Nikos Skalkotos committed
66
67
%(prog)s will work with the packages that are declared in `devflow.conf'
file, which must exist in the top-level directory of the git repository.
68

Christos Stavrakakis's avatar
Christos Stavrakakis committed
69
70
71
72
73
74
"""


def print_help(prog):
    print DESCRIPTION % {"prog": prog}

Vangelis Koukis's avatar
Vangelis Koukis committed
75
76

def main():
77
    from devflow.version import __version__  # pylint: disable=E0611,F0401
Vangelis Koukis's avatar
Vangelis Koukis committed
78
    parser = OptionParser(usage="usage: %prog [options] mode",
Christos Stavrakakis's avatar
Christos Stavrakakis committed
79
80
81
82
83
84
                          version="devflow %s" % __version__,
                          add_help_option=False)
    parser.add_option("-h", "--help",
                      action="store_true",
                      default=False,
                      help="show this help message")
Vangelis Koukis's avatar
Vangelis Koukis committed
85
86
87
88
89
90
91
92
    parser.add_option("-k", "--keep-repo",
                      action="store_true",
                      dest="keep_repo",
                      default=False,
                      help="Do not delete the cloned repository")
    parser.add_option("-b", "--build-dir",
                      dest="build_dir",
                      default=None,
Nikos Skalkotos's avatar
Nikos Skalkotos committed
93
                      help="Directory to store created packages")
Vangelis Koukis's avatar
Vangelis Koukis committed
94
95
96
97
98
99
100
101
102
    parser.add_option("-r", "--repo-dir",
                      dest="repo_dir",
                      default=None,
                      help="Directory to clone repository")
    parser.add_option("-d", "--dirty",
                      dest="force_dirty",
                      default=False,
                      action="store_true",
                      help="Do not check if working directory is dirty")
103
104
105
    parser.add_option("-c", "--config-file",
                      dest="config_file",
                      help="Override default configuration file")
Christos Stavrakakis's avatar
Christos Stavrakakis committed
106
107
108
109
110
    parser.add_option("--no-sign",
                      dest="sign",
                      action="store_false",
                      default=True,
                      help="Do not sign the packages")
111
112
113
    parser.add_option("--key-id",
                      dest="keyid",
                      help="Use this keyid for gpg signing")
114
115
    parser.add_option("--dist",
                      dest="dist",
116
                      default=None,
Christos Stavrakakis's avatar
Christos Stavrakakis committed
117
                      help="Force distribution in Debian changelog")
118
119
120
121
122
123
    parser.add_option("-S", "--source-only",
                      dest="source_only",
                      default=False,
                      action="store_true",
                      help="Specifies a source-only build, no binary packages"
                           " need to be made.")
124
125
126
127
128
    parser.add_option("--debian-branch",
                      dest="debian_branch",
                      default=None,
                      help="Use this debian branch, instead of"
                           "auto-discovering the debian branch to use")
129
130
131
132
133
    parser.add_option("--push-back",
                      dest="push_back",
                      default=False,
                      action="store_true",
                      help="Automatically push branches and tags to repo.")
134
135
136
137
138
    parser.add_option("--color",
                      dest="color_output",
                      default="auto",
                      help="Enable/disable colored output. Default mode is" +
                           " auto, available options are yes/no")
Vangelis Koukis's avatar
Vangelis Koukis committed
139
140
141

    (options, args) = parser.parse_args()

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
    if options.color_output == "yes":
        use_colors = True
    elif options.color_output == "no":
        use_colors = False
    else:
        if sys.stdout.isatty():
            use_colors = True
        else:
            use_colors = False

    red = lambda x: x
    green = lambda x: x

    if use_colors:
        try:
            import colors
            red = colors.red
            green = colors.green
        except AttributeError:
            pass

    print_red = lambda x: sys.stdout.write(red(x) + "\n")
    print_green = lambda x: sys.stdout.write(green(x) + "\n")

Christos Stavrakakis's avatar
Christos Stavrakakis committed
166
167
168
169
170
    if options.help:
        print_help(parser.get_prog_name())
        parser.print_help()
        return

171
    # Get build mode
Christos Stavrakakis's avatar
Christos Stavrakakis committed
172
173
174
    try:
        mode = args[0]
    except IndexError:
175
        mode = utils.get_build_mode()
Vangelis Koukis's avatar
Vangelis Koukis committed
176
177
178
179
    if mode not in AVAILABLE_MODES:
        raise ValueError(red("Invalid argument! Mode must be one: %s"
                         % ", ".join(AVAILABLE_MODES)))

180
    # Load the repository
181
    original_repo = utils.get_repository()
Vangelis Koukis's avatar
Vangelis Koukis committed
182

183
    # Check that repository is clean
Christos Stavrakakis's avatar
Christos Stavrakakis committed
184
    toplevel = original_repo.working_dir
Vangelis Koukis's avatar
Vangelis Koukis committed
185
186
187
    if original_repo.is_dirty() and not options.force_dirty:
        raise RuntimeError(red("Repository %s is dirty." % toplevel))

188
    # Get packages from configuration file
189
    config = utils.get_config(options.config_file)
190
    packages = config['packages'].keys()
Christos Stavrakakis's avatar
Christos Stavrakakis committed
191
    print_green("Will build the following packages:\n" + "\n".join(packages))
Vangelis Koukis's avatar
Vangelis Koukis committed
192

Christos Stavrakakis's avatar
Christos Stavrakakis committed
193
    # Get current branch name and type and check if it is a valid one
194
    branch = original_repo.head.reference.name
195
    branch = utils.undebianize(branch)
Christos Stavrakakis's avatar
Fix bug    
Christos Stavrakakis committed
196
    branch_type_str = utils.get_branch_type(branch)
Vangelis Koukis's avatar
Vangelis Koukis committed
197

Christos Stavrakakis's avatar
Christos Stavrakakis committed
198
    if branch_type_str not in BRANCH_TYPES.keys():
Christos Stavrakakis's avatar
Christos Stavrakakis committed
199
        allowed_branches = ", ".join(BRANCH_TYPES.keys())
200
201
202
        raise ValueError("Malformed branch name '%s', cannot classify as"
                         " one of %s" % (branch, allowed_branches))

203
    # Fix needed environment variables
204
    v = utils.get_vcs_info()
205
    os.environ["DEVFLOW_BUILD_MODE"] = mode
206
207
    os.environ["DEBFULLNAME"] = v.name
    os.environ["DEBEMAIL"] = v.email
208

209
210
211
    # Check that base version file and branch are correct
    versioning.get_python_version()

212
    # Get the debian branch
213
214
215
216
    if options.debian_branch:
        debian_branch = options.debian_branch
    else:
        debian_branch = utils.get_debian_branch(branch)
217
    origin_debian = "origin/" + debian_branch
218
219
220

    # Clone the repo
    repo_dir = options.repo_dir or create_temp_directory("df-repo")
Christos Stavrakakis's avatar
Christos Stavrakakis committed
221
    repo_dir = os.path.abspath(repo_dir)
222
    repo = original_repo.clone(repo_dir, branch=branch)
223
    print_green("Cloned repository to '%s'." % repo_dir)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
224

Christos Stavrakakis's avatar
Christos Stavrakakis committed
225
226
227
228
    build_dir = options.build_dir or create_temp_directory("df-build")
    build_dir = os.path.abspath(build_dir)
    print_green("Build directory: '%s'" % build_dir)

229
    # Create the debian branch
Christos Stavrakakis's avatar
Christos Stavrakakis committed
230
    repo.git.branch(debian_branch, origin_debian)
231
232
    print_green("Created branch '%s' to track '%s'" % (debian_branch,
                origin_debian))
Vangelis Koukis's avatar
Vangelis Koukis committed
233

234
    # Go to debian branch
Vangelis Koukis's avatar
Vangelis Koukis committed
235
236
237
    repo.git.checkout(debian_branch)
    print_green("Changed to branch '%s'" % debian_branch)

238
    # Merge with starting branch
Vangelis Koukis's avatar
Vangelis Koukis committed
239
    repo.git.merge(branch)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
240
    print_green("Merged branch '%s' into '%s'" % (branch, debian_branch))
Vangelis Koukis's avatar
Vangelis Koukis committed
241

242
    # Compute python and debian version
Vangelis Koukis's avatar
Vangelis Koukis committed
243
    cd(repo_dir)
244
    python_version = versioning.get_python_version()
Christos Stavrakakis's avatar
Christos Stavrakakis committed
245
    debian_version = versioning.\
Christos Stavrakakis's avatar
Christos Stavrakakis committed
246
        debian_version_from_python_version(python_version)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
247
    print_green("The new debian version will be: '%s'" % debian_version)
Vangelis Koukis's avatar
Vangelis Koukis committed
248

249
    # Update the version files
Christos Stavrakakis's avatar
Christos Stavrakakis committed
250
    versioning.update_version()
251

Christos Stavrakakis's avatar
Christos Stavrakakis committed
252
253
254
255
256
257
258
259
260
    if not options.sign:
        sign_tag_opt = None
    elif options.keyid:
        sign_tag_opt = "-u=%s" % options.keyid
    elif mode == "release":
        sign_tag_opt = "-s"
    else:
        sign_tag_opt = None

Christos Stavrakakis's avatar
Christos Stavrakakis committed
261
262
    # Tag branch with python version
    branch_tag = python_version
Christos Stavrakakis's avatar
Christos Stavrakakis committed
263
    tag_message = "%s version %s" % (mode.capitalize(), python_version)
264
    try:
Christos Stavrakakis's avatar
Christos Stavrakakis committed
265
        repo.git.tag(branch_tag, branch, sign_tag_opt, "-m %s" % tag_message)
266
267
268
    except GitCommandError:
        # Tag may already exist, if only the debian branch has changed
        pass
269
270
    upstream_tag = "upstream/" + branch_tag
    repo.git.tag(upstream_tag, branch)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
271

272
    # Update changelog
Vangelis Koukis's avatar
Vangelis Koukis committed
273
    dch = git_dch("--debian-branch=%s" % debian_branch,
Christos Stavrakakis's avatar
Christos Stavrakakis committed
274
275
276
277
278
                  "--git-author",
                  "--ignore-regex=\".*\"",
                  "--multimaint-merge",
                  "--since=HEAD",
                  "--new-version=%s" % debian_version)
Vangelis Koukis's avatar
Vangelis Koukis committed
279
280
    print_green("Successfully ran '%s'" % " ".join(dch.cmd))

281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
    if options.dist is not None:
        distribution = options.dist
    elif mode == "release":
        distribution = utils.get_distribution_codename()
    else:
        distribution = "unstable"

    f = open("debian/changelog", 'r+')
    lines = f.readlines()
    lines[0] = lines[0].replace("UNRELEASED", distribution)
    lines[2] = lines[2].replace("UNRELEASED", "%s build" % mode)
    f.seek(0)
    f.writelines(lines)
    f.close()

Vangelis Koukis's avatar
Vangelis Koukis committed
296
    if mode == "release":
297
        call("vim debian/changelog")
Christos Stavrakakis's avatar
Christos Stavrakakis committed
298
299
300
301

    # Add changelog to INDEX
    repo.git.add("debian/changelog")
    # Commit Changes
302
303
    repo.git.commit("-s", "debian/changelog",
                    m="Bump version to %s" % debian_version)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
304
    # Tag debian branch
305
    debian_branch_tag = "debian/" + utils.version_to_tag(debian_version)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
306
    tag_message = "%s version %s" % (mode.capitalize(), debian_version)
307
    if mode == "release":
Christos Stavrakakis's avatar
Christos Stavrakakis committed
308
        repo.git.tag(debian_branch_tag, sign_tag_opt, "-m %s" % tag_message)
Vangelis Koukis's avatar
Vangelis Koukis committed
309

Christos Stavrakakis's avatar
Christos Stavrakakis committed
310
    # Create debian packages
Christos Stavrakakis's avatar
Christos Stavrakakis committed
311
    cd(repo_dir)
312
313
    version_files = []
    for _, pkg_info in config['packages'].items():
314
315
        if pkg_info.get("version_file"):
            version_files.extend(pkg_info.as_list('version_file'))
316
317

    # Add version.py files to repo
Christos Stavrakakis's avatar
Christos Stavrakakis committed
318
    repo.git.add("-f", *version_files)
319

320
321
    # Export version info to debuilg environment
    os.environ["DEB_DEVFLOW_DEBIAN_VERSION"] = debian_version
322
    os.environ["DEB_DEVFLOW_VERSION"] = python_version
Christos Stavrakakis's avatar
Christos Stavrakakis committed
323
324
325
    build_cmd = "git-buildpackage --git-export-dir=%s"\
                " --git-upstream-branch=%s --git-debian-branch=%s"\
                " --git-export=INDEX --git-ignore-new -sa"\
326
                " --source-option=--auto-commit"\
327
                " --git-upstream-tag=%s"\
328
                % (build_dir, branch, debian_branch, upstream_tag)
329
330
    if options.source_only:
        build_cmd += " -S"
Christos Stavrakakis's avatar
Christos Stavrakakis committed
331
332
    if not options.sign:
        build_cmd += " -uc -us"
333
334
    elif options.keyid:
        build_cmd += " -k\"'%s'\"" % options.keyid
Christos Stavrakakis's avatar
Christos Stavrakakis committed
335
    call(build_cmd)
Vangelis Koukis's avatar
Vangelis Koukis committed
336

337
    # Remove cloned repo
338
    if mode != 'release' and not options.keep_repo:
Vangelis Koukis's avatar
Vangelis Koukis committed
339
340
341
        print_green("Removing cloned repo '%s'." % repo_dir)
        rm("-r", repo_dir)

Christos Stavrakakis's avatar
Christos Stavrakakis committed
342
343
344
345
346
347
348
349
350
    # Print final info
    info = (("Version", debian_version),
            ("Upstream branch", branch),
            ("Upstream tag", branch_tag),
            ("Debian branch", debian_branch),
            ("Debian tag", debian_branch_tag),
            ("Repository directory", repo_dir),
            ("Packages directory", build_dir))
    print_green("\n".join(["%s: %s" % (name, val) for name, val in info]))
Christos Stavrakakis's avatar
Christos Stavrakakis committed
351

352
    # Print help message
Christos Stavrakakis's avatar
Christos Stavrakakis committed
353
    if mode == "release":
Christos Stavrakakis's avatar
Christos Stavrakakis committed
354
355
356
357
        origin = original_repo.remote().url
        repo.create_remote("original_origin", origin)
        print_green("Created remote 'original_origin' for the repository '%s'"
                    % origin)
Christos Stavrakakis's avatar
Christos Stavrakakis committed
358

Christos Stavrakakis's avatar
Christos Stavrakakis committed
359
360
361
        print_green("To update repositories '%s' and '%s' go to '%s' and run:"
                    % (toplevel, origin, repo_dir))
        for remote in ['origin', 'original_origin']:
362
363
            objects = [debian_branch, branch_tag, debian_branch_tag]
            print_green("git push %s %s" % (remote, " ".join(objects)))
364
365
366
367
        if options.push_back:
            objects = [debian_branch, branch_tag, debian_branch_tag]
            repo.git.push("origin", *objects)
            print_green("Automatically updated origin repo.")
Christos Stavrakakis's avatar
Christos Stavrakakis committed
368
369


370
371
372
def create_temp_directory(suffix):
    create_dir_cmd = mktemp("-d", "/tmp/" + suffix + "-XXXXX")
    return create_dir_cmd.stdout.strip()
Vangelis Koukis's avatar
Vangelis Koukis committed
373
374


375
376
377
def call(cmd):
    rc = os.system(cmd)
    if rc:
Christos Stavrakakis's avatar
Christos Stavrakakis committed
378
        raise RuntimeError("Command '%s' failed!" % cmd)
379
380


Vangelis Koukis's avatar
Vangelis Koukis committed
381
382
if __name__ == "__main__":
    sys.exit(main())