qa_job.py 4.14 KB
Newer Older
1
2
3
#
#

4
# Copyright (C) 2012, 2014 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Job-related QA tests.

"""

Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
26
import functools
27
28
import re

29
30
31
from ganeti.utils import retry
from ganeti import constants
from ganeti import query
32
33
import qa_config
import qa_error
34
import qa_job_utils
35
import qa_utils
36
37
from qa_utils import AssertCommand, GetCommandOutput

38
39
40

def TestJobList():
  """gnt-job list"""
41
42
  qa_utils.GenericQueryTest("gnt-job", query.JOB_FIELDS.keys(),
                            namefield="id", test_unknown=False)
43
44
45
46
47


def TestJobListFields():
  """gnt-node list-fields"""
  qa_utils.GenericQueryFieldsTest("gnt-job", query.JOB_FIELDS.keys())
48
49


Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
50
51
52
53
54
55
56
57
58
59
def _GetJobStatus(job_id):
  """ Retrieves the status of a job.

  @type job_id: string
  @param job_id: The job id, represented as a string.
  @rtype: string or None

  @return: The job status, or None if not present.

  """
60
  return qa_job_utils.GetJobStatuses([job_id]).get(job_id, None)
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80


def _RetryingFetchJobStatus(retry_status, job_id):
  """ Used with C{retry.Retry}, waits for a status other than the one given.

  @type retry_status: string
  @param retry_status: The old job status, expected to change.
  @type job_id: string
  @param job_id: The job id, represented as a string.

  @rtype: string or None
  @return: The new job status, or None if none could be retrieved.

  """
  status = _GetJobStatus(job_id)
  if status == retry_status:
    raise retry.RetryAgain()
  return status


81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
def TestJobCancellation():
  """gnt-job cancel"""
  # The delay used for the first command should be large enough for the next
  # command and the cancellation command to complete before the first job is
  # done. The second delay should be small enough that not too much time is
  # spend waiting in the case of a failed cancel and a running command.
  FIRST_COMMAND_DELAY = 10.0
  AssertCommand(["gnt-debug", "delay", "--submit", str(FIRST_COMMAND_DELAY)])

  SECOND_COMMAND_DELAY = 1.0
  master = qa_config.GetMasterNode()

  # Forcing tty usage does not work on buildbot, so force all output of this
  # command to be redirected to stdout
  job_id_output = GetCommandOutput(
    master.primary, "gnt-debug delay --submit %s 2>&1" % SECOND_COMMAND_DELAY
  )

  possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output)
  if len(possible_job_ids) != 1:
    raise qa_error.Error("Cannot parse gnt-debug delay output to find job id")

  job_id = possible_job_ids[0]
  AssertCommand(["gnt-job", "cancel", job_id])

  # Now wait until the second job finishes, and expect the watch to fail due to
  # job cancellation
  AssertCommand(["gnt-job", "watch", job_id], fail=True)

  # Then check for job cancellation
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
111
112
113
114
115
116
117
118
  job_status = _GetJobStatus(job_id)
  if job_status != constants.JOB_STATUS_CANCELED:
    # Try and see if the job is being cancelled, and wait until the status
    # changes or we hit a timeout
    if job_status == constants.JOB_STATUS_CANCELING:
      retry_fn = functools.partial(_RetryingFetchJobStatus,
                                   constants.JOB_STATUS_CANCELING, job_id)
      try:
119
120
121
122
123
        # The multiplier to use is arbitrary, setting it higher could prevent
        # flakiness
        WAIT_MULTIPLIER = 4.0
        job_status = retry.Retry(retry_fn, 2.0,
                                 WAIT_MULTIPLIER * FIRST_COMMAND_DELAY)
Hrvoje Ribicic's avatar
Hrvoje Ribicic committed
124
125
126
127
128
129
130
      except retry.RetryTimeout:
        # The job status remains the same
        pass

    if job_status != constants.JOB_STATUS_CANCELED:
      raise qa_error.Error("Job was not successfully cancelled, status "
                           "found: %s" % job_status)