Commit 580b1fdd authored by Jose A. Lopes's avatar Jose A. Lopes

Hook h2spy in Makefile.am

* add rules to Makefile.am to use hs2py to generate the Python opcodes
  from Haskell and update tests to check that Haskell and Python contain
  the same opcodes.
* split 'opcodes.py' in 'opcodes.py.in_after' and 'opcodes_base.py',
  moving as much code as possible to 'opcodes_base.py' without
  creating a circular dependency
* update reference in the remaining Python modules
* remove lib/opcodes.py dependency from documentation target in order
  to prevent recompilation of documentation in the distributed source
  tarball, in particular because 'sphinx-build' is an optional
  dependency (issue 547)
Signed-off-by: default avatarJose A. Lopes <jabolopes@google.com>
Reviewed-by: default avatarGuido Trotter <ultrotter@google.com>
parent 4157b044
......@@ -242,7 +242,8 @@ built_base_sources = \
built_python_base_sources = \
lib/_autoconf.py \
lib/_vcsversion.py
lib/_vcsversion.py \
lib/opcodes.py
BUILT_PYTHON_SOURCES = \
$(built_python_base_sources) \
......@@ -291,7 +292,7 @@ pkgpython_PYTHON = \
lib/mcpu.py \
lib/netutils.py \
lib/objects.py \
lib/opcodes.py \
lib/opcodes_base.py \
lib/outils.py \
lib/ovf.py \
lib/pathutils.py \
......@@ -506,6 +507,7 @@ HS_COMPILE_PROGS= \
src/ganeti-mond \
src/hconfd \
src/hluxid \
src/hs2py \
src/rpc-test
# All Haskell non-test programs to be compiled but not automatically installed
......@@ -603,6 +605,7 @@ HS_LIB_SRCS = \
src/Ganeti/Hypervisor/Xen/Types.hs \
src/Ganeti/Hash.hs \
src/Ganeti/Hs2Py/GenOpCodes.hs \
src/Ganeti/Hs2Py/OpDoc.hs \
src/Ganeti/JQueue.hs \
src/Ganeti/JSON.hs \
src/Ganeti/Jobs.hs \
......@@ -695,7 +698,9 @@ HS_BUILT_SRCS = \
src/Ganeti/Version.hs
HS_BUILT_SRCS_IN = \
$(patsubst %,%.in,$(filter-out src/Ganeti/Curl/Internal.hs,$(HS_BUILT_SRCS))) \
src/Ganeti/Curl/Internal.hsc
src/Ganeti/Curl/Internal.hsc \
lib/opcodes.py.in_after \
lib/opcodes.py.in_before
HS_LIBTESTBUILT_SRCS = $(HS_LIBTEST_SRCS) $(HS_BUILT_SRCS)
......@@ -712,7 +717,7 @@ doc/man-html/index.html: doc/manpages-enabled.rst $(mandocrst)
# it changes
doc/html/index.html doc/man-html/index.html: $(docinput) doc/conf.py \
configure.ac $(RUN_IN_TEMPDIR) lib/build/sphinx_ext.py \
lib/build/shell_example_lexer.py lib/opcodes.py lib/ht.py \
lib/build/shell_example_lexer.py lib/ht.py \
doc/css/style.css lib/rapi/connector.py lib/rapi/rlib2.py \
autotools/sphinx-wrapper | $(BUILT_PYTHON_SOURCES)
@test -n "$(SPHINX)" || \
......@@ -1666,6 +1671,13 @@ lib/_vcsversion.py: Makefile vcs-version | stamp-directories
echo "VCS_VERSION = '$$VCSVER'"; \
} > $@
lib/opcodes.py: Makefile src/hs2py src/Ganeti/Constants.hs \
lib/opcodes.py.in_before lib/opcodes.py.in_after \
| stamp-directories
cat $(abs_top_srcdir)/lib/opcodes.py.in_before > $@
src/hs2py >> $@
cat $(abs_top_srcdir)/lib/opcodes.py.in_after >> $@
lib/_generated_rpc.py: lib/rpc_defs.py $(BUILD_RPC)
PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(BUILD_RPC) lib/rpc_defs.py > $@
......
......@@ -28,7 +28,7 @@ mv $tmpdir/lib $tmpdir/ganeti
ln -T -s $tmpdir/ganeti $tmpdir/lib
mkdir -p $tmpdir/src $tmpdir/test/hs
for hfile in htools ganeti-confd mon-collector; do
for hfile in htools ganeti-confd mon-collector hs2py; do
if [ -e src/$hfile ]; then
ln -s $PWD/src/$hfile $tmpdir/src/
fi
......
......@@ -51,6 +51,7 @@ from ganeti import compat
from ganeti import errors
from ganeti import utils
from ganeti import opcodes
from ganeti import opcodes_base
from ganeti import ht
from ganeti import rapi
from ganeti import luxi
......@@ -83,7 +84,7 @@ def _GetCommonParamNames():
names = set(map(compat.fst, opcodes.OpCode.OP_PARAMS))
# The "depends" attribute should be listed
names.remove(opcodes.DEPEND_ATTR)
names.remove(opcodes_base.DEPEND_ATTR)
return names
......
......@@ -49,6 +49,7 @@ from ganeti import serializer
from ganeti import workerpool
from ganeti import locking
from ganeti import opcodes
from ganeti import opcodes_base
from ganeti import errors
from ganeti import mcpu
from ganeti import utils
......@@ -231,7 +232,7 @@ class _QueuedJob(object):
count = 0
for queued_op in self.ops:
op = queued_op.input
reason_src = opcodes.NameToReasonSrc(op.__class__.__name__)
reason_src = opcodes_base.NameToReasonSrc(op.__class__.__name__)
reason_text = "job=%d;index=%d" % (self.id, count)
reason = getattr(op, "reason", [])
reason.append((reason_src, reason_text, utils.EpochNano()))
......@@ -910,7 +911,7 @@ class _OpExecContext:
self.summary = op.input.Summary()
# Create local copy to modify
if getattr(op.input, opcodes.DEPEND_ATTR, None):
if getattr(op.input, opcodes_base.DEPEND_ATTR, None):
self.jobdeps = op.input.depends[:]
else:
self.jobdeps = None
......@@ -2196,11 +2197,11 @@ class JobQueue(object):
" are %s" % (idx, op.priority, allowed))
# Check job dependencies
dependencies = getattr(op.input, opcodes.DEPEND_ATTR, None)
if not opcodes.TNoRelativeJobDependencies(dependencies):
dependencies = getattr(op.input, opcodes_base.DEPEND_ATTR, None)
if not opcodes_base.TNoRelativeJobDependencies(dependencies):
raise errors.GenericError("Opcode %s has invalid dependencies, must"
" match %s: %s" %
(idx, opcodes.TNoRelativeJobDependencies,
(idx, opcodes_base.TNoRelativeJobDependencies,
dependencies))
# Write to disk
......@@ -2298,7 +2299,7 @@ class JobQueue(object):
for (idx, (job_id, ops)) in enumerate(zip(job_ids, jobs)):
for op in ops:
if getattr(op, opcodes.DEPEND_ATTR, None):
if getattr(op, opcodes_base.DEPEND_ATTR, None):
(status, data) = \
self._ResolveJobDependencies(compat.partial(resolve_fn, idx),
op.depends)
......
......@@ -36,6 +36,7 @@ import itertools
import traceback
from ganeti import opcodes
from ganeti import opcodes_base
from ganeti import constants
from ganeti import errors
from ganeti import hooksmaster
......@@ -207,7 +208,7 @@ def _SetBaseOpParams(src, defcomment, dst):
hasattr(src, "priority")):
dst.priority = src.priority
if not getattr(dst, opcodes.COMMENT_ATTR, None):
if not getattr(dst, opcodes_base.COMMENT_ATTR, None):
dst.comment = defcomment
......
This diff is collapsed.
def _GetOpList():
"""Returns list of all defined opcodes.
Does not eliminate duplicates by C{OP_ID}.
"""
return [v for v in globals().values()
if (isinstance(v, type) and issubclass(v, OpCode) and
hasattr(v, "OP_ID") and v is not OpCode and
v.OP_ID != 'OP_INSTANCE_MULTI_ALLOC_BASE')]
OP_MAPPING = dict((v.OP_ID, v) for v in _GetOpList())
#
#
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""OpCodes module
Note that this file is autogenerated using @src/hs2py@ with a header
from @lib/opcodes.py.in_before@ and a footer from @lib/opcodes.py.in_after@.
This module implements part of the data structures which define the
cluster operations - the so-called opcodes.
Every operation which modifies the cluster state is expressed via
opcodes.
"""
# this are practically structures, so disable the message about too
# few public methods:
# pylint: disable=R0903
# pylint: disable=C0301
from ganeti import constants
from ganeti import ht
from ganeti import opcodes_base
class OpCode(opcodes_base.BaseOpCode):
"""Abstract OpCode.
This is the root of the actual OpCode hierarchy. All clases derived
from this class should override OP_ID.
@cvar OP_ID: The ID of this opcode. This should be unique amongst all
children of this class.
@cvar OP_DSC_FIELD: The name of a field whose value will be included in the
string returned by Summary(); see the docstring of that
method for details).
@cvar OP_DSC_FORMATTER: A callable that should format the OP_DSC_FIELD; if
not present, then the field will be simply converted
to string
@cvar OP_PARAMS: List of opcode attributes, the default values they should
get if not already defined, and types they must match.
@cvar OP_RESULT: Callable to verify opcode result
@cvar WITH_LU: Boolean that specifies whether this should be included in
mcpu's dispatch table
@ivar dry_run: Whether the LU should be run in dry-run mode, i.e. just
the check steps
@ivar priority: Opcode priority for queue
"""
# pylint: disable=E1101
# as OP_ID is dynamically defined
WITH_LU = True
OP_PARAMS = [
("dry_run", None, ht.TMaybe(ht.TBool), "Run checks only, don't execute"),
("debug_level", None, ht.TMaybe(ht.TNonNegative(ht.TInt)), "Debug level"),
("priority", constants.OP_PRIO_DEFAULT,
ht.TElemOf(constants.OP_PRIO_SUBMIT_VALID), "Opcode priority"),
(opcodes_base.DEPEND_ATTR, None, opcodes_base.BuildJobDepCheck(True),
"Job dependencies; if used through ``SubmitManyJobs`` relative (negative)"
" job IDs can be used; see :doc:`design document <design-chained-jobs>`"
" for details"),
(opcodes_base.COMMENT_ATTR, None, ht.TMaybe(ht.TString),
"Comment describing the purpose of the opcode"),
(constants.OPCODE_REASON, None, ht.TMaybe(ht.TListOf(ht.TAny)),
"The reason trail, describing why the OpCode is executed"),
]
OP_RESULT = None
def __getstate__(self):
"""Specialized getstate for opcodes.
This method adds to the state dictionary the OP_ID of the class,
so that on unload we can identify the correct class for
instantiating the opcode.
@rtype: C{dict}
@return: the state as a dictionary
"""
data = opcodes_base.BaseOpCode.__getstate__(self)
data["OP_ID"] = self.OP_ID
return data
@classmethod
def LoadOpCode(cls, data):
"""Generic load opcode method.
The method identifies the correct opcode class from the dict-form
by looking for a OP_ID key, if this is not found, or its value is
not available in this module as a child of this class, we fail.
@type data: C{dict}
@param data: the serialized opcode
"""
if not isinstance(data, dict):
raise ValueError("Invalid data to LoadOpCode (%s)" % type(data))
if "OP_ID" not in data:
raise ValueError("Invalid data to LoadOpcode, missing OP_ID")
op_id = data["OP_ID"]
op_class = None
if op_id in OP_MAPPING:
op_class = OP_MAPPING[op_id]
else:
raise ValueError("Invalid data to LoadOpCode: OP_ID %s unsupported" %
op_id)
op = op_class()
new_data = data.copy()
del new_data["OP_ID"]
op.__setstate__(new_data)
return op
def Summary(self):
"""Generates a summary description of this opcode.
The summary is the value of the OP_ID attribute (without the "OP_"
prefix), plus the value of the OP_DSC_FIELD attribute, if one was
defined; this field should allow to easily identify the operation
(for an instance creation job, e.g., it would be the instance
name).
"""
assert self.OP_ID is not None and len(self.OP_ID) > 3
# all OP_ID start with OP_, we remove that
txt = self.OP_ID[3:]
field_name = getattr(self, "OP_DSC_FIELD", None)
if field_name:
field_value = getattr(self, field_name, None)
field_formatter = getattr(self, "OP_DSC_FORMATTER", None)
if callable(field_formatter):
field_value = field_formatter(field_value)
elif isinstance(field_value, (list, tuple)):
field_value = ",".join(str(i) for i in field_value)
txt = "%s(%s)" % (txt, field_value)
return txt
def TinySummary(self):
"""Generates a compact summary description of the opcode.
"""
assert self.OP_ID.startswith("OP_")
text = self.OP_ID[3:]
for (prefix, supplement) in opcodes_base.SUMMARY_PREFIX.items():
if text.startswith(prefix):
return supplement + text[len(prefix):]
return text
class OpInstanceMultiAllocBase(OpCode):
"""Allocates multiple instances.
"""
def __getstate__(self):
"""Generic serializer.
"""
state = OpCode.__getstate__(self)
if hasattr(self, "instances"):
# pylint: disable=E1101
state["instances"] = [inst.__getstate__() for inst in self.instances]
return state
def __setstate__(self, state):
"""Generic unserializer.
This method just restores from the serialized state the attributes
of the current instance.
@param state: the serialized opcode data
@type state: C{dict}
"""
if not isinstance(state, dict):
raise ValueError("Invalid data to __setstate__: expected dict, got %s" %
type(state))
if "instances" in state:
state["instances"] = map(OpCode.LoadOpCode, state["instances"])
return OpCode.__setstate__(self, state)
def Validate(self, set_defaults):
"""Validates this opcode.
We do this recursively.
"""
OpCode.Validate(self, set_defaults)
for inst in self.instances: # pylint: disable=E1101
inst.Validate(set_defaults)
#
#
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""OpCodes base module
This module implements part of the data structures which define the
cluster operations - the so-called opcodes.
Every operation which modifies the cluster state is expressed via
opcodes.
"""
# this are practically structures, so disable the message about too
# few public methods:
# pylint: disable=R0903
import copy
import logging
import re
from ganeti import constants
from ganeti import errors
from ganeti import ht
from ganeti import outils
#: OP_ID conversion regular expression
_OPID_RE = re.compile("([a-z])([A-Z])")
SUMMARY_PREFIX = {
"CLUSTER_": "C_",
"GROUP_": "G_",
"NODE_": "N_",
"INSTANCE_": "I_",
}
#: Attribute name for dependencies
DEPEND_ATTR = "depends"
#: Attribute name for comment
COMMENT_ATTR = "comment"
def _NameComponents(name):
"""Split an opcode class name into its components
@type name: string
@param name: the class name, as OpXxxYyy
@rtype: array of strings
@return: the components of the name
"""
assert name.startswith("Op")
# Note: (?<=[a-z])(?=[A-Z]) would be ideal, since it wouldn't
# consume any input, and hence we would just have all the elements
# in the list, one by one; but it seems that split doesn't work on
# non-consuming input, hence we have to process the input string a
# bit
name = _OPID_RE.sub(r"\1,\2", name)
elems = name.split(",")
return elems
def _NameToId(name):
"""Convert an opcode class name to an OP_ID.
@type name: string
@param name: the class name, as OpXxxYyy
@rtype: string
@return: the name in the OP_XXXX_YYYY format
"""
if not name.startswith("Op"):
return None
return "_".join(n.upper() for n in _NameComponents(name))
def NameToReasonSrc(name):
"""Convert an opcode class name to a source string for the reason trail
@type name: string
@param name: the class name, as OpXxxYyy
@rtype: string
@return: the name in the OP_XXXX_YYYY format
"""
if not name.startswith("Op"):
return None
return "%s:%s" % (constants.OPCODE_REASON_SRC_OPCODE,
"_".join(n.lower() for n in _NameComponents(name)))
class _AutoOpParamSlots(outils.AutoSlots):
"""Meta class for opcode definitions.
"""
def __new__(mcs, name, bases, attrs):
"""Called when a class should be created.
@param mcs: The meta class
@param name: Name of created class
@param bases: Base classes
@type attrs: dict
@param attrs: Class attributes
"""
assert "OP_ID" not in attrs, "Class '%s' defining OP_ID" % name
slots = mcs._GetSlots(attrs)
assert "OP_DSC_FIELD" not in attrs or attrs["OP_DSC_FIELD"] in slots, \
"Class '%s' uses unknown field in OP_DSC_FIELD" % name
assert ("OP_DSC_FORMATTER" not in attrs or
callable(attrs["OP_DSC_FORMATTER"])), \
("Class '%s' uses non-callable in OP_DSC_FORMATTER (%s)" %
(name, type(attrs["OP_DSC_FORMATTER"])))
attrs["OP_ID"] = _NameToId(name)
return outils.AutoSlots.__new__(mcs, name, bases, attrs)
@classmethod
def _GetSlots(mcs, attrs):
"""Build the slots out of OP_PARAMS.
"""
# Always set OP_PARAMS to avoid duplicates in BaseOpCode.GetAllParams
params = attrs.setdefault("OP_PARAMS", [])
# Use parameter names as slots
return [pname for (pname, _, _, _) in params]
class BaseOpCode(outils.ValidatedSlots):
"""A simple serializable object.
This object serves as a parent class for OpCode without any custom
field handling.
"""
# pylint: disable=E1101
# as OP_ID is dynamically defined
__metaclass__ = _AutoOpParamSlots
def __getstate__(self):
"""Generic serializer.
This method just returns the contents of the instance as a
dictionary.
@rtype: C{dict}
@return: the instance attributes and their values
"""
state = {}
for name in self.GetAllSlots():
if hasattr(self, name):
state[name] = getattr(self, name)
return state
def __setstate__(self, state):
"""Generic unserializer.
This method just restores from the serialized state the attributes
of the current instance.
@param state: the serialized opcode data
@type state: C{dict}
"""
if not isinstance(state, dict):
raise ValueError("Invalid data to __setstate__: expected dict, got %s" %
type(state))
for name in self.GetAllSlots():
if name not in state and hasattr(self, name):
delattr(self, name)
for name in state:
setattr(self, name, state[name])
@classmethod
def GetAllParams(cls):
"""Compute list of all parameters for an opcode.
"""
slots = []
for parent in cls.__mro__:
slots.extend(getattr(parent, "OP_PARAMS", []))
return slots
def Validate(self, set_defaults): # pylint: disable=W0221
"""Validate opcode parameters, optionally setting default values.
@type set_defaults: bool
@param set_defaults: Whether to set default values
@raise errors.OpPrereqError: When a parameter value doesn't match
requirements
"""
for (attr_name, default, test, _) in self.GetAllParams():
assert callable(test)
if hasattr(self, attr_name):
attr_val = getattr(self, attr_name)
else:
attr_val = copy.deepcopy(default)
if test(attr_val):
if set_defaults:
setattr(self, attr_name, attr_val)
elif ht.TInt(attr_val) and test(float(attr_val)):
if set_defaults:
setattr(self, attr_name, float(attr_val))
else:
logging.error("OpCode %s, parameter %s, has invalid type %s/value"
" '%s' expecting type %s",
self.OP_ID, attr_name, type(attr_val), attr_val, test)
if attr_val is None:
logging.error("OpCode %s, parameter %s, has default value None which"
" is does not check against the parameter's type: this"
" means this parameter is required but no value was"
" given",
self.OP_ID, attr_name)
raise errors.OpPrereqError("Parameter '%s.%s' fails validation" %
(self.OP_ID, attr_name),
errors.ECODE_INVAL)
def BuildJobDepCheck(relative):
"""Builds check for job dependencies (L{DEPEND_ATTR}).
@type relative: bool
@param relative: Whether to accept relative job IDs (negative)
@rtype: callable
"""
if relative:
job_id = ht.TOr(ht.TJobId, ht.TRelativeJobId)
else:
job_id = ht.TJobId
job_dep = \
ht.TAnd(ht.TOr(ht.TListOf(ht.TAny), ht.TTuple),
ht.TIsLength(2),
ht.TItems([job_id,
ht.TListOf(ht.TElemOf(constants.JOBS_FINALIZED))]))
return ht.TMaybe(ht.TListOf(job_dep))
TNoRelativeJobDependencies = BuildJobDepCheck(False)
def RequireSharedFileStorage():
"""Checks that shared file storage is enabled.
While it doesn't really fit into this module, L{utils} was deemed too large