baserlib.py 17.4 KB
Newer Older
1
2
3
#
#

Iustin Pop's avatar
Iustin Pop committed
4
# Copyright (C) 2006, 2007, 2008, 2012 Google Inc.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Remote API base resources library.

"""

26
# pylint: disable=C0103
Iustin Pop's avatar
Iustin Pop committed
27
28
29

# C0103: Invalid name, since the R_* names are not conforming

Iustin Pop's avatar
Iustin Pop committed
30
31
import logging

32
from ganeti import luxi
33
import ganeti.rpc.errors as rpcerr
34
from ganeti import rapi
35
from ganeti import http
Iustin Pop's avatar
Iustin Pop committed
36
from ganeti import errors
37
from ganeti import compat
38
from ganeti import constants
39
from ganeti import pathutils
40
from ganeti import utils
41

42

43
44
45
# Dummy value to detect unchanged parameters
_DEFAULT = object()

46
#: Supported HTTP methods
47
_SUPPORTED_METHODS = compat.UniqueFrozenset([
48
49
50
51
52
53
  http.HTTP_DELETE,
  http.HTTP_GET,
  http.HTTP_POST,
  http.HTTP_PUT,
  ])

54

55
56
57
58
59
def _BuildOpcodeAttributes():
  """Builds list of attributes used for per-handler opcodes.

  """
  return [(method, "%s_OPCODE" % method, "%s_RENAME" % method,
60
           "%s_ALIASES" % method, "Get%sOpInput" % method.capitalize())
61
62
63
          for method in _SUPPORTED_METHODS]


64
OPCODE_ATTRS = _BuildOpcodeAttributes()
65
66


67
68
69
def BuildUriList(ids, uri_format, uri_fields=("name", "uri")):
  """Builds a URI list as used by index resources.

Iustin Pop's avatar
Iustin Pop committed
70
71
72
  @param ids: list of ids as strings
  @param uri_format: format to be applied for URI
  @param uri_fields: optional parameter for field IDs
73
74
75

  """
  (field_id, field_uri) = uri_fields
76

77
  def _MapId(m_id):
Michael Hanselmann's avatar
Michael Hanselmann committed
78
79
80
81
    return {
      field_id: m_id,
      field_uri: uri_format % m_id,
      }
82
83
84
85
86
87
88
89
90
91
92

  # Make sure the result is sorted, makes it nicer to look at and simplifies
  # unittests.
  ids.sort()

  return map(_MapId, ids)


def MapFields(names, data):
  """Maps two lists into one dictionary.

Iustin Pop's avatar
Iustin Pop committed
93
94
95
  Example::
      >>> MapFields(["a", "b"], ["foo", 123])
      {'a': 'foo', 'b': 123}
96

Iustin Pop's avatar
Iustin Pop committed
97
98
  @param names: field names (list of strings)
  @param data: field data (list)
99
100
101
102

  """
  if len(names) != len(data):
    raise AttributeError("Names and data must have the same length")
103
  return dict(zip(names, data))
104
105


106
107
108
def MapBulkFields(itemslist, fields):
  """Map value to field name in to one dictionary.

Iustin Pop's avatar
Iustin Pop committed
109
110
111
112
  @param itemslist: a list of items values
  @param fields: a list of items names

  @return: a list of mapped dictionaries
113
114
115
116
117
118
119
120
121

  """
  items_details = []
  for item in itemslist:
    mapped = MapFields(fields, item)
    items_details.append(mapped)
  return items_details


122
def FillOpcode(opcls, body, static, rename=None):
123
124
125
126
127
128
129
130
131
132
  """Fills an opcode with body parameters.

  Parameter types are checked.

  @type opcls: L{opcodes.OpCode}
  @param opcls: Opcode class
  @type body: dict
  @param body: Body parameters as received from client
  @type static: dict
  @param static: Static parameters which can't be modified by client
133
134
  @type rename: dict
  @param rename: Renamed parameters, key as old name, value as new name
135
136
137
  @return: Opcode object

  """
138
139
140
141
  if body is None:
    params = {}
  else:
    CheckType(body, dict, "Body contents")
142

143
144
    # Make copy to be modified
    params = body.copy()
145
146
147
148
149
150
151
152
153
154
155

  if rename:
    for old, new in rename.items():
      if new in params and old in params:
        raise http.HttpBadRequest("Parameter '%s' was renamed to '%s', but"
                                  " both are specified" %
                                  (old, new))
      if old in params:
        assert new not in params
        params[new] = params.pop(old)

156
  if static:
157
    overwritten = set(params.keys()) & set(static.keys())
158
159
160
161
162
163
164
165
166
167
    if overwritten:
      raise http.HttpBadRequest("Can't overwrite static parameters %r" %
                                overwritten)

    params.update(static)

  # Convert keys to strings (simplejson decodes them as unicode)
  params = dict((str(key), value) for (key, value) in params.items())

  try:
168
    op = opcls(**params) # pylint: disable=W0142
169
170
171
172
173
174
175
    op.Validate(False)
  except (errors.OpPrereqError, TypeError), err:
    raise http.HttpBadRequest("Invalid body parameters: %s" % err)

  return op


176
177
178
179
180
def HandleItemQueryErrors(fn, *args, **kwargs):
  """Converts errors when querying a single item.

  """
  try:
181
    result = fn(*args, **kwargs)
182
183
184
185
186
187
  except errors.OpPrereqError, err:
    if len(err.args) == 2 and err.args[1] == errors.ECODE_NOENT:
      raise http.HttpNotFound()

    raise

188
189
190
191
192
193
  # In case split query mechanism is used
  if not result:
    raise http.HttpNotFound()

  return result

194

195
196
def FeedbackFn(msg):
  """Feedback logging function for jobs.
Iustin Pop's avatar
Iustin Pop committed
197
198
199
200

  We don't have a stdout for printing log messages, so log them to the
  http log at least.

201
  @param msg: the message
202

Iustin Pop's avatar
Iustin Pop committed
203
  """
204
  (_, log_type, log_msg) = msg
Iustin Pop's avatar
Iustin Pop committed
205
206
207
  logging.info("%s: %s", log_type, log_msg)


208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
def CheckType(value, exptype, descr):
  """Abort request if value type doesn't match expected type.

  @param value: Value
  @type exptype: type
  @param exptype: Expected type
  @type descr: string
  @param descr: Description of value
  @return: Value (allows inline usage)

  """
  if not isinstance(value, exptype):
    raise http.HttpBadRequest("%s: Type is '%s', but '%s' is expected" %
                              (descr, type(value).__name__, exptype.__name__))

  return value


def CheckParameter(data, name, default=_DEFAULT, exptype=_DEFAULT):
  """Check and return the value for a given parameter.

  If no default value was given and the parameter doesn't exist in the input
  data, an error is raise.

  @type data: dict
  @param data: Dictionary containing input data
  @type name: string
  @param name: Parameter name
  @param default: Default value (can be None)
  @param exptype: Expected type (can be None)

  """
  try:
    value = data[name]
  except KeyError:
    if default is not _DEFAULT:
      return default

    raise http.HttpBadRequest("Required parameter '%s' is missing" %
                              name)

  if exptype is _DEFAULT:
    return value

  return CheckType(value, exptype, "'%s' parameter" % name)


255
class ResourceBase(object):
256
257
258
  """Generic class for resources.

  """
259
260
261
262
263
264
  # Default permission requirements
  GET_ACCESS = []
  PUT_ACCESS = [rapi.RAPI_ACCESS_WRITE]
  POST_ACCESS = [rapi.RAPI_ACCESS_WRITE]
  DELETE_ACCESS = [rapi.RAPI_ACCESS_WRITE]

265
  def __init__(self, items, queryargs, req, _client_cls=None):
266
267
    """Generic resource constructor.

Iustin Pop's avatar
Iustin Pop committed
268
269
    @param items: a list with variables encoded in the URL
    @param queryargs: a dictionary with additional options from URL
270
271
    @param req: Request context
    @param _client_cls: L{luxi} client class (unittests only)
272
273

    """
Michele Tartara's avatar
Michele Tartara committed
274
275
    assert isinstance(queryargs, dict)

276
277
    self.items = items
    self.queryargs = queryargs
278
    self._req = req
279
280
281
282

    if _client_cls is None:
      _client_cls = luxi.Client

283
    self._client_cls = _client_cls
284

285
286
287
288
289
290
291
  def _GetRequestBody(self):
    """Returns the body data.

    """
    return self._req.private.body_data

  request_body = property(fget=_GetRequestBody)
292

293
  def _checkIntVariable(self, name, default=0):
294
295
296
    """Return the parsed value of an int argument.

    """
297
    val = self.queryargs.get(name, default)
298
299
300
301
    if isinstance(val, list):
      if val:
        val = val[0]
      else:
302
        val = default
303
304
    try:
      val = int(val)
Michael Hanselmann's avatar
Michael Hanselmann committed
305
    except (ValueError, TypeError):
Iustin Pop's avatar
Iustin Pop committed
306
      raise http.HttpBadRequest("Invalid value for the"
307
308
309
                                " '%s' parameter" % (name,))
    return val

310
  def _checkStringVariable(self, name, default=None):
311
    """Return the parsed value of a string argument.
312
313
314
315
316
317
318
319
320

    """
    val = self.queryargs.get(name, default)
    if isinstance(val, list):
      if val:
        val = val[0]
      else:
        val = default
    return val
321

Iustin Pop's avatar
Iustin Pop committed
322
323
324
325
326
327
328
329
330
  def getBodyParameter(self, name, *args):
    """Check and return the value for a given parameter.

    If a second parameter is not given, an error will be returned,
    otherwise this parameter specifies the default value.

    @param name: the required parameter

    """
331
    if args:
Luca Bigliardi's avatar
Luca Bigliardi committed
332
      return CheckParameter(self.request_body, name, default=args[0])
333

Luca Bigliardi's avatar
Luca Bigliardi committed
334
    return CheckParameter(self.request_body, name)
Iustin Pop's avatar
Iustin Pop committed
335

336
337
338
339
  def useLocking(self):
    """Check if the request specifies locking.

    """
340
    return bool(self._checkIntVariable("lock"))
341
342
343
344
345

  def useBulk(self):
    """Check if the request specifies bulk querying.

    """
346
    return bool(self._checkIntVariable("bulk"))
Iustin Pop's avatar
Iustin Pop committed
347

348
349
350
351
  def useForce(self):
    """Check if the request specifies a forced operation.

    """
352
    return bool(self._checkIntVariable("force"))
353

Iustin Pop's avatar
Iustin Pop committed
354
355
356
357
  def dryRun(self):
    """Check if the request specifies dry-run mode.

    """
358
    return bool(self._checkIntVariable("dry-run"))
359

360
  def GetClient(self, query=False):
361
362
    """Wrapper for L{luxi.Client} with HTTP-specific error handling.

363
364
365
366
367
    @param query: this signifies that the client will only be used for
        queries; if the build-time parameter enable-split-queries is
        enabled, then the client will be connected to the query socket
        instead of the masterd socket

368
    """
369
    if query:
370
      address = pathutils.QUERY_SOCKET
371
372
    else:
      address = None
373
374
    # Could be a function, pylint: disable=R0201
    try:
375
      return self._client_cls(address=address)
376
    except rpcerr.NoMasterError, err:
377
      raise http.HttpBadGateway("Can't connect to master daemon: %s" % err)
378
    except rpcerr.PermissionError:
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
      raise http.HttpInternalServerError("Internal error: no permission to"
                                         " connect to the master daemon")

  def SubmitJob(self, op, cl=None):
    """Generic wrapper for submit job, for better http compatibility.

    @type op: list
    @param op: the list of opcodes for the job
    @type cl: None or luxi.Client
    @param cl: optional luxi client to use
    @rtype: string
    @return: the job ID

    """
    if cl is None:
      cl = self.GetClient()
    try:
      return cl.SubmitJob(op)
    except errors.JobQueueFull:
      raise http.HttpServiceUnavailable("Job queue is full, needs archiving")
    except errors.JobQueueDrainError:
      raise http.HttpServiceUnavailable("Job queue is drained, cannot submit")
401
    except rpcerr.NoMasterError, err:
402
      raise http.HttpBadGateway("Master seems to be unreachable: %s" % err)
403
    except rpcerr.PermissionError:
404
405
      raise http.HttpInternalServerError("Internal error: no permission to"
                                         " connect to the master daemon")
406
    except rpcerr.TimeoutError, err:
407
408
      raise http.HttpGatewayTimeout("Timeout while talking to the master"
                                    " daemon: %s" % err)
409
410


411
412
413
414
415
def GetResourceOpcodes(cls):
  """Returns all opcodes used by a resource.

  """
  return frozenset(filter(None, (getattr(cls, op_attr, None)
416
                                 for (_, op_attr, _, _, _) in OPCODE_ATTRS)))
417
418


419
420
421
422
423
424
425
426
427
428
429
def GetHandlerAccess(handler, method):
  """Returns the access rights for a method on a handler.

  @type handler: L{ResourceBase}
  @type method: string
  @rtype: string or None

  """
  return getattr(handler, "%s_ACCESS" % method, None)


430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
def GetHandler(get_fn, aliases):
  result = get_fn()
  if not isinstance(result, dict) or aliases is None:
    return result

  for (param, alias) in aliases.items():
    if param in result:
      if alias in result:
        raise http.HttpBadRequest("Parameter '%s' has an alias of '%s', but"
                                  " both values are present in response" %
                                  (param, alias))
      result[alias] = result[param]

  return result


446
447
448
449
450
451
452
453
454
455
456
class _MetaOpcodeResource(type):
  """Meta class for RAPI resources.

  """
  def __call__(mcs, *args, **kwargs):
    """Instantiates class and patches it for use by the RAPI daemon.

    """
    # Access to private attributes of a client class, pylint: disable=W0212
    obj = type.__call__(mcs, *args, **kwargs)

457
    for (method, op_attr, rename_attr, aliases_attr, fn_attr) in OPCODE_ATTRS:
458
      if hasattr(obj, method):
459
460
461
        # If the method handler is already defined, "*_RENAME" or
        # "Get*OpInput" shouldn't be (they're only used by the automatically
        # generated handler)
462
463
        assert not hasattr(obj, rename_attr)
        assert not hasattr(obj, fn_attr)
464
465
466
467
468
469
470
471
472
473

        # The aliases are allowed only on GET calls
        assert not hasattr(obj, aliases_attr) or method == http.HTTP_GET

        # GET methods can add aliases of values they return under a different
        # name
        if method == http.HTTP_GET and hasattr(obj, aliases_attr):
          setattr(obj, method,
                  compat.partial(GetHandler, getattr(obj, method),
                                 getattr(obj, aliases_attr)))
474
475
476
477
478
479
480
481
482
483
484
      else:
        # Try to generate handler method on handler instance
        try:
          opcode = getattr(obj, op_attr)
        except AttributeError:
          pass
        else:
          setattr(obj, method,
                  compat.partial(obj._GenericHandler, opcode,
                                 getattr(obj, rename_attr, None),
                                 getattr(obj, fn_attr, obj._GetDefaultData)))
485
486
487
488
489
490
491
492
493
494
495
496

    return obj


class OpcodeResource(ResourceBase):
  """Base class for opcode-based RAPI resources.

  Instances of this class automatically gain handler functions through
  L{_MetaOpcodeResource} for any method for which a C{$METHOD$_OPCODE} variable
  is defined at class level. Subclasses can define a C{Get$Method$OpInput}
  method to do their own opcode input processing (e.g. for static values). The
  C{$METHOD$_RENAME} variable defines which values are renamed (see
Iustin Pop's avatar
Iustin Pop committed
497
  L{baserlib.FillOpcode}).
498
499
500
501
502
503
  Still default behavior cannot be totally overriden. There are opcode params
  that are available to all opcodes, e.g. "depends". In case those params
  (currently only "depends") are found in the original request's body, they are
  added to the dictionary of parsed parameters and eventually passed to the
  opcode. If the parsed body is not represented as a dictionary object, the
  values are not added.
504
505
506
507

  @cvar GET_OPCODE: Set this to a class derived from L{opcodes.OpCode} to
    automatically generate a GET handler submitting the opcode
  @cvar GET_RENAME: Set this to rename parameters in the GET handler (see
Iustin Pop's avatar
Iustin Pop committed
508
    L{baserlib.FillOpcode})
509
510
  @cvar GET_ALIASES: Set this to duplicate return values in GET results (see
    L{baserlib.GetHandler})
511
512
513
514
515
516
  @ivar GetGetOpInput: Define this to override the default method for
    getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData})

  @cvar PUT_OPCODE: Set this to a class derived from L{opcodes.OpCode} to
    automatically generate a PUT handler submitting the opcode
  @cvar PUT_RENAME: Set this to rename parameters in the PUT handler (see
Iustin Pop's avatar
Iustin Pop committed
517
    L{baserlib.FillOpcode})
518
519
520
521
522
  @ivar GetPutOpInput: Define this to override the default method for
    getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData})

  @cvar POST_OPCODE: Set this to a class derived from L{opcodes.OpCode} to
    automatically generate a POST handler submitting the opcode
523
  @cvar POST_RENAME: Set this to rename parameters in the POST handler (see
Iustin Pop's avatar
Iustin Pop committed
524
    L{baserlib.FillOpcode})
525
526
527
528
  @ivar GetPostOpInput: Define this to override the default method for
    getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData})

  @cvar DELETE_OPCODE: Set this to a class derived from L{opcodes.OpCode} to
529
    automatically generate a DELETE handler submitting the opcode
530
  @cvar DELETE_RENAME: Set this to rename parameters in the DELETE handler (see
Iustin Pop's avatar
Iustin Pop committed
531
    L{baserlib.FillOpcode})
532
533
534
535
536
537
538
539
540
  @ivar GetDeleteOpInput: Define this to override the default method for
    getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData})

  """
  __metaclass__ = _MetaOpcodeResource

  def _GetDefaultData(self):
    return (self.request_body, None)

541
542
543
544
545
546
547
548
  def _GetRapiOpName(self):
    """Extracts the name of the RAPI operation from the class name

    """
    if self.__class__.__name__.startswith("R_2_"):
      return self.__class__.__name__[4:]
    return self.__class__.__name__

549
550
551
  def _GetCommonStatic(self):
    """Return the static parameters common to all the RAPI calls

552
553
554
555
556
    The reason is a parameter present in all the RAPI calls, and the reason
    trail has to be build for all of them, so the parameter is read here and
    used to build the reason trail, that is the actual parameter passed
    forward.

557
    """
558
559
560
561
562
563
564
565
566
567
568
569
    trail = []
    usr_reason = self._checkStringVariable("reason", default=None)
    if usr_reason:
      trail.append((constants.OPCODE_REASON_SRC_USER,
                    usr_reason,
                    utils.EpochNano()))
    reason_src = "%s:%s" % (constants.OPCODE_REASON_SRC_RLIB2,
                            self._GetRapiOpName())
    trail.append((reason_src, "", utils.EpochNano()))
    common_static = {
      "reason": trail,
      }
570
571
    return common_static

572
573
574
575
576
577
578
579
  def _GetDepends(self):
    ret = {}
    if isinstance(self.request_body, dict):
      depends = self.getBodyParameter("depends", None)
      if depends:
        ret.update({"depends": depends})
    return ret

580
  def _GenericHandler(self, opcode, rename, fn):
581
    (body, specific_static) = fn()
582
583
    if isinstance(body, dict):
      body.update(self._GetDepends())
584
585
586
    static = self._GetCommonStatic()
    if specific_static:
      static.update(specific_static)
587
588
    op = FillOpcode(opcode, body, static, rename=rename)
    return self.SubmitJob([op])