Hook h2spy in Makefile.am

* add rules to Makefile.am to use hs2py to generate the Python opcodes
  from Haskell and update tests to check that Haskell and Python contain
  the same opcodes.
* split 'opcodes.py' in 'opcodes.py.in_after' and 'opcodes_base.py',
  moving as much code as possible to 'opcodes_base.py' without
  creating a circular dependency
* update reference in the remaining Python modules
* remove lib/opcodes.py dependency from documentation target in order
  to prevent recompilation of documentation in the distributed source
  tarball, in particular because 'sphinx-build' is an optional
  dependency (issue 547)
Signed-off-by: Jose A. Lopes <jabolopes@google.com>
Reviewed-by: Guido Trotter <ultrotter@google.com>

Makefile.am

@@ -242,7 +242,8 @@ built_base_sources = \
 
 built_python_base_sources = \
 	lib/_autoconf.py \
-	lib/_vcsversion.py
+	lib/_vcsversion.py \
+	lib/opcodes.py
 
 BUILT_PYTHON_SOURCES = \
 	$(built_python_base_sources) \
@@ -291,7 +292,7 @@ pkgpython_PYTHON = \
 	lib/mcpu.py \
 	lib/netutils.py \
 	lib/objects.py \
-	lib/opcodes.py \
+	lib/opcodes_base.py \
 	lib/outils.py \
 	lib/ovf.py \
 	lib/pathutils.py \
@@ -506,6 +507,7 @@ HS_COMPILE_PROGS= \
 	src/ganeti-mond \
 	src/hconfd \
 	src/hluxid \
+	src/hs2py \
 	src/rpc-test
 
 # All Haskell non-test programs to be compiled but not automatically installed
@@ -603,6 +605,7 @@ HS_LIB_SRCS = \
 	src/Ganeti/Hypervisor/Xen/Types.hs \
 	src/Ganeti/Hash.hs \
 	src/Ganeti/Hs2Py/GenOpCodes.hs \
+	src/Ganeti/Hs2Py/OpDoc.hs \
 	src/Ganeti/JQueue.hs \
 	src/Ganeti/JSON.hs \
 	src/Ganeti/Jobs.hs \
@@ -695,7 +698,9 @@ HS_BUILT_SRCS = \
 	src/Ganeti/Version.hs
 HS_BUILT_SRCS_IN = \
 	$(patsubst %,%.in,$(filter-out src/Ganeti/Curl/Internal.hs,$(HS_BUILT_SRCS))) \
-	src/Ganeti/Curl/Internal.hsc
+	src/Ganeti/Curl/Internal.hsc \
+	lib/opcodes.py.in_after \
+	lib/opcodes.py.in_before
 
 HS_LIBTESTBUILT_SRCS = $(HS_LIBTEST_SRCS) $(HS_BUILT_SRCS)
 
@@ -712,7 +717,7 @@ doc/man-html/index.html: doc/manpages-enabled.rst $(mandocrst)
 # it changes
 doc/html/index.html doc/man-html/index.html: $(docinput) doc/conf.py \
 	configure.ac $(RUN_IN_TEMPDIR) lib/build/sphinx_ext.py \
-	lib/build/shell_example_lexer.py lib/opcodes.py lib/ht.py \
+	lib/build/shell_example_lexer.py lib/ht.py \
 	doc/css/style.css lib/rapi/connector.py lib/rapi/rlib2.py \
 	autotools/sphinx-wrapper | $(BUILT_PYTHON_SOURCES)
 	@test -n "$(SPHINX)" || \
@@ -1666,6 +1671,13 @@ lib/_vcsversion.py: Makefile vcs-version | stamp-directories
 	  echo "VCS_VERSION = '$$VCSVER'"; \
 	} > $@
 
+lib/opcodes.py: Makefile src/hs2py src/Ganeti/Constants.hs \
+		lib/opcodes.py.in_before lib/opcodes.py.in_after \
+		| stamp-directories
+	cat $(abs_top_srcdir)/lib/opcodes.py.in_before > $@
+	src/hs2py >> $@
+	cat $(abs_top_srcdir)/lib/opcodes.py.in_after >> $@
+
 lib/_generated_rpc.py: lib/rpc_defs.py $(BUILD_RPC)
 	PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(BUILD_RPC) lib/rpc_defs.py > $@
 

autotools/run-in-tempdir

@@ -28,7 +28,7 @@ mv $tmpdir/lib $tmpdir/ganeti
 ln -T -s $tmpdir/ganeti $tmpdir/lib
 
 mkdir -p $tmpdir/src $tmpdir/test/hs
-for hfile in htools ganeti-confd mon-collector; do
+for hfile in htools ganeti-confd mon-collector hs2py; do
   if [ -e src/$hfile ]; then
     ln -s $PWD/src/$hfile $tmpdir/src/
   fi

lib/build/sphinx_ext.py

@@ -51,6 +51,7 @@ from ganeti import compat
 from ganeti import errors
 from ganeti import utils
 from ganeti import opcodes
+from ganeti import opcodes_base
 from ganeti import ht
 from ganeti import rapi
 from ganeti import luxi
@@ -83,7 +84,7 @@ def _GetCommonParamNames():
   names = set(map(compat.fst, opcodes.OpCode.OP_PARAMS))
 
   # The "depends" attribute should be listed
-  names.remove(opcodes.DEPEND_ATTR)
+  names.remove(opcodes_base.DEPEND_ATTR)
 
   return names
 

lib/jqueue.py

@@ -49,6 +49,7 @@ from ganeti import serializer
 from ganeti import workerpool
 from ganeti import locking
 from ganeti import opcodes
+from ganeti import opcodes_base
 from ganeti import errors
 from ganeti import mcpu
 from ganeti import utils
@@ -231,7 +232,7 @@ class _QueuedJob(object):
     count = 0
     for queued_op in self.ops:
       op = queued_op.input
-      reason_src = opcodes.NameToReasonSrc(op.__class__.__name__)
+      reason_src = opcodes_base.NameToReasonSrc(op.__class__.__name__)
       reason_text = "job=%d;index=%d" % (self.id, count)
       reason = getattr(op, "reason", [])
       reason.append((reason_src, reason_text, utils.EpochNano()))
@@ -910,7 +911,7 @@ class _OpExecContext:
     self.summary = op.input.Summary()
 
     # Create local copy to modify
-    if getattr(op.input, opcodes.DEPEND_ATTR, None):
+    if getattr(op.input, opcodes_base.DEPEND_ATTR, None):
       self.jobdeps = op.input.depends[:]
     else:
       self.jobdeps = None
@@ -2196,11 +2197,11 @@ class JobQueue(object):
                                   " are %s" % (idx, op.priority, allowed))
 
       # Check job dependencies
-      dependencies = getattr(op.input, opcodes.DEPEND_ATTR, None)
-      if not opcodes.TNoRelativeJobDependencies(dependencies):
+      dependencies = getattr(op.input, opcodes_base.DEPEND_ATTR, None)
+      if not opcodes_base.TNoRelativeJobDependencies(dependencies):
         raise errors.GenericError("Opcode %s has invalid dependencies, must"
                                   " match %s: %s" %
-                                  (idx, opcodes.TNoRelativeJobDependencies,
+                                  (idx, opcodes_base.TNoRelativeJobDependencies,
                                    dependencies))
 
     # Write to disk
@@ -2298,7 +2299,7 @@ class JobQueue(object):
 
     for (idx, (job_id, ops)) in enumerate(zip(job_ids, jobs)):
       for op in ops:
-        if getattr(op, opcodes.DEPEND_ATTR, None):
+        if getattr(op, opcodes_base.DEPEND_ATTR, None):
           (status, data) = \
             self._ResolveJobDependencies(compat.partial(resolve_fn, idx),
                                          op.depends)

lib/mcpu.py

@@ -36,6 +36,7 @@ import itertools
 import traceback
 
 from ganeti import opcodes
+from ganeti import opcodes_base
 from ganeti import constants
 from ganeti import errors
 from ganeti import hooksmaster
@@ -207,7 +208,7 @@ def _SetBaseOpParams(src, defcomment, dst):
       hasattr(src, "priority")):
     dst.priority = src.priority
 
-  if not getattr(dst, opcodes.COMMENT_ATTR, None):
+  if not getattr(dst, opcodes_base.COMMENT_ATTR, None):
     dst.comment = defcomment
 
 

lib/opcodes.py.in_after

+
+
+def _GetOpList():
+  """Returns list of all defined opcodes.
+
+  Does not eliminate duplicates by C{OP_ID}.
+
+  """
+  return [v for v in globals().values()
+          if (isinstance(v, type) and issubclass(v, OpCode) and
+              hasattr(v, "OP_ID") and v is not OpCode and
+              v.OP_ID != 'OP_INSTANCE_MULTI_ALLOC_BASE')]
+
+
+OP_MAPPING = dict((v.OP_ID, v) for v in _GetOpList())

lib/opcodes.py.in_before

+#
+#
+
+# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""OpCodes module
+
+Note that this file is autogenerated using @src/hs2py@ with a header
+from @lib/opcodes.py.in_before@ and a footer from @lib/opcodes.py.in_after@.
+
+This module implements part of the data structures which define the
+cluster operations - the so-called opcodes.
+
+Every operation which modifies the cluster state is expressed via
+opcodes.
+
+"""
+
+# this are practically structures, so disable the message about too
+# few public methods:
+# pylint: disable=R0903
+# pylint: disable=C0301
+
+from ganeti import constants
+from ganeti import ht
+
+from ganeti import opcodes_base
+
+
+class OpCode(opcodes_base.BaseOpCode):
+  """Abstract OpCode.
+
+  This is the root of the actual OpCode hierarchy. All clases derived
+  from this class should override OP_ID.
+
+  @cvar OP_ID: The ID of this opcode. This should be unique amongst all
+               children of this class.
+  @cvar OP_DSC_FIELD: The name of a field whose value will be included in the
+                      string returned by Summary(); see the docstring of that
+                      method for details).
+  @cvar OP_DSC_FORMATTER: A callable that should format the OP_DSC_FIELD; if
+                          not present, then the field will be simply converted
+                          to string
+  @cvar OP_PARAMS: List of opcode attributes, the default values they should
+                   get if not already defined, and types they must match.
+  @cvar OP_RESULT: Callable to verify opcode result
+  @cvar WITH_LU: Boolean that specifies whether this should be included in
+      mcpu's dispatch table
+  @ivar dry_run: Whether the LU should be run in dry-run mode, i.e. just
+                 the check steps
+  @ivar priority: Opcode priority for queue
+
+  """
+  # pylint: disable=E1101
+  # as OP_ID is dynamically defined
+  WITH_LU = True
+  OP_PARAMS = [
+    ("dry_run", None, ht.TMaybe(ht.TBool), "Run checks only, don't execute"),
+    ("debug_level", None, ht.TMaybe(ht.TNonNegative(ht.TInt)), "Debug level"),
+    ("priority", constants.OP_PRIO_DEFAULT,
+     ht.TElemOf(constants.OP_PRIO_SUBMIT_VALID), "Opcode priority"),
+    (opcodes_base.DEPEND_ATTR, None, opcodes_base.BuildJobDepCheck(True),
+     "Job dependencies; if used through ``SubmitManyJobs`` relative (negative)"
+     " job IDs can be used; see :doc:`design document <design-chained-jobs>`"
+     " for details"),
+    (opcodes_base.COMMENT_ATTR, None, ht.TMaybe(ht.TString),
+     "Comment describing the purpose of the opcode"),
+    (constants.OPCODE_REASON, None, ht.TMaybe(ht.TListOf(ht.TAny)),
+     "The reason trail, describing why the OpCode is executed"),
+    ]
+  OP_RESULT = None
+
+  def __getstate__(self):
+    """Specialized getstate for opcodes.
+
+    This method adds to the state dictionary the OP_ID of the class,
+    so that on unload we can identify the correct class for
+    instantiating the opcode.
+
+    @rtype:   C{dict}
+    @return:  the state as a dictionary
+
+    """
+    data = opcodes_base.BaseOpCode.__getstate__(self)
+    data["OP_ID"] = self.OP_ID
+    return data
+
+  @classmethod
+  def LoadOpCode(cls, data):
+    """Generic load opcode method.
+
+    The method identifies the correct opcode class from the dict-form
+    by looking for a OP_ID key, if this is not found, or its value is
+    not available in this module as a child of this class, we fail.
+
+    @type data:  C{dict}
+    @param data: the serialized opcode
+
+    """
+    if not isinstance(data, dict):
+      raise ValueError("Invalid data to LoadOpCode (%s)" % type(data))
+    if "OP_ID" not in data:
+      raise ValueError("Invalid data to LoadOpcode, missing OP_ID")
+    op_id = data["OP_ID"]
+    op_class = None
+    if op_id in OP_MAPPING:
+      op_class = OP_MAPPING[op_id]
+    else:
+      raise ValueError("Invalid data to LoadOpCode: OP_ID %s unsupported" %
+                       op_id)
+    op = op_class()
+    new_data = data.copy()
+    del new_data["OP_ID"]
+    op.__setstate__(new_data)
+    return op
+
+  def Summary(self):
+    """Generates a summary description of this opcode.
+
+    The summary is the value of the OP_ID attribute (without the "OP_"
+    prefix), plus the value of the OP_DSC_FIELD attribute, if one was
+    defined; this field should allow to easily identify the operation
+    (for an instance creation job, e.g., it would be the instance
+    name).
+
+    """
+    assert self.OP_ID is not None and len(self.OP_ID) > 3
+    # all OP_ID start with OP_, we remove that
+    txt = self.OP_ID[3:]
+    field_name = getattr(self, "OP_DSC_FIELD", None)
+    if field_name:
+      field_value = getattr(self, field_name, None)
+      field_formatter = getattr(self, "OP_DSC_FORMATTER", None)
+      if callable(field_formatter):
+        field_value = field_formatter(field_value)
+      elif isinstance(field_value, (list, tuple)):
+        field_value = ",".join(str(i) for i in field_value)
+      txt = "%s(%s)" % (txt, field_value)
+    return txt
+
+  def TinySummary(self):
+    """Generates a compact summary description of the opcode.
+
+    """
+    assert self.OP_ID.startswith("OP_")
+
+    text = self.OP_ID[3:]
+
+    for (prefix, supplement) in opcodes_base.SUMMARY_PREFIX.items():
+      if text.startswith(prefix):
+        return supplement + text[len(prefix):]
+
+    return text
+
+
+class OpInstanceMultiAllocBase(OpCode):
+  """Allocates multiple instances.
+
+  """
+  def __getstate__(self):
+    """Generic serializer.
+
+    """
+    state = OpCode.__getstate__(self)
+    if hasattr(self, "instances"):
+      # pylint: disable=E1101
+      state["instances"] = [inst.__getstate__() for inst in self.instances]
+    return state
+
+  def __setstate__(self, state):
+    """Generic unserializer.
+
+    This method just restores from the serialized state the attributes
+    of the current instance.
+
+    @param state: the serialized opcode data
+    @type state: C{dict}
+
+    """
+    if not isinstance(state, dict):
+      raise ValueError("Invalid data to __setstate__: expected dict, got %s" %
+                       type(state))
+
+    if "instances" in state:
+      state["instances"] = map(OpCode.LoadOpCode, state["instances"])
+
+    return OpCode.__setstate__(self, state)
+
+  def Validate(self, set_defaults):
+    """Validates this opcode.
+
+    We do this recursively.
+
+    """
+    OpCode.Validate(self, set_defaults)
+
+    for inst in self.instances: # pylint: disable=E1101
+      inst.Validate(set_defaults)

lib/opcodes_base.py

+#
+#
+
+# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""OpCodes base module
+
+This module implements part of the data structures which define the
+cluster operations - the so-called opcodes.
+
+Every operation which modifies the cluster state is expressed via
+opcodes.
+
+"""
+
+# this are practically structures, so disable the message about too
+# few public methods:
+# pylint: disable=R0903
+
+import copy
+import logging
+import re
+
+from ganeti import constants
+from ganeti import errors
+from ganeti import ht
+from ganeti import outils
+
+
+#: OP_ID conversion regular expression
+_OPID_RE = re.compile("([a-z])([A-Z])")
+
+SUMMARY_PREFIX = {
+  "CLUSTER_": "C_",
+  "GROUP_": "G_",
+  "NODE_": "N_",
+  "INSTANCE_": "I_",
+  }
+
+#: Attribute name for dependencies
+DEPEND_ATTR = "depends"
+
+#: Attribute name for comment
+COMMENT_ATTR = "comment"
+
+
+def _NameComponents(name):
+  """Split an opcode class name into its components
+
+  @type name: string
+  @param name: the class name, as OpXxxYyy
+  @rtype: array of strings
+  @return: the components of the name
+
+  """
+  assert name.startswith("Op")
+  # Note: (?<=[a-z])(?=[A-Z]) would be ideal, since it wouldn't
+  # consume any input, and hence we would just have all the elements
+  # in the list, one by one; but it seems that split doesn't work on
+  # non-consuming input, hence we have to process the input string a
+  # bit
+  name = _OPID_RE.sub(r"\1,\2", name)
+  elems = name.split(",")
+  return elems
+
+
+def _NameToId(name):
+  """Convert an opcode class name to an OP_ID.
+
+  @type name: string
+  @param name: the class name, as OpXxxYyy
+  @rtype: string
+  @return: the name in the OP_XXXX_YYYY format
+
+  """
+  if not name.startswith("Op"):
+    return None
+  return "_".join(n.upper() for n in _NameComponents(name))
+
+
+def NameToReasonSrc(name):
+  """Convert an opcode class name to a source string for the reason trail
+
+  @type name: string
+  @param name: the class name, as OpXxxYyy
+  @rtype: string
+  @return: the name in the OP_XXXX_YYYY format
+
+  """
+  if not name.startswith("Op"):
+    return None
+  return "%s:%s" % (constants.OPCODE_REASON_SRC_OPCODE,
+                    "_".join(n.lower() for n in _NameComponents(name)))
+
+
+class _AutoOpParamSlots(outils.AutoSlots):
+  """Meta class for opcode definitions.
+
+  """
+  def __new__(mcs, name, bases, attrs):
+    """Called when a class should be created.
+
+    @param mcs: The meta class
+    @param name: Name of created class
+    @param bases: Base classes
+    @type attrs: dict
+    @param attrs: Class attributes
+
+    """
+    assert "OP_ID" not in attrs, "Class '%s' defining OP_ID" % name
+
+    slots = mcs._GetSlots(attrs)
+    assert "OP_DSC_FIELD" not in attrs or attrs["OP_DSC_FIELD"] in slots, \
+      "Class '%s' uses unknown field in OP_DSC_FIELD" % name
+    assert ("OP_DSC_FORMATTER" not in attrs or
+            callable(attrs["OP_DSC_FORMATTER"])), \
+      ("Class '%s' uses non-callable in OP_DSC_FORMATTER (%s)" %
+       (name, type(attrs["OP_DSC_FORMATTER"])))
+
+    attrs["OP_ID"] = _NameToId(name)
+
+    return outils.AutoSlots.__new__(mcs, name, bases, attrs)
+
+  @classmethod
+  def _GetSlots(mcs, attrs):
+    """Build the slots out of OP_PARAMS.
+
+    """
+    # Always set OP_PARAMS to avoid duplicates in BaseOpCode.GetAllParams
+    params = attrs.setdefault("OP_PARAMS", [])
+
+    # Use parameter names as slots
+    return [pname for (pname, _, _, _) in params]
+
+
+class BaseOpCode(outils.ValidatedSlots):
+  """A simple serializable object.
+
+  This object serves as a parent class for OpCode without any custom
+  field handling.
+
+  """
+  # pylint: disable=E1101
+  # as OP_ID is dynamically defined
+  __metaclass__ = _AutoOpParamSlots
+
+  def __getstate__(self):
+    """Generic serializer.
+
+    This method just returns the contents of the instance as a
+    dictionary.
+
+    @rtype:  C{dict}
+    @return: the instance attributes and their values
+
+    """
+    state = {}
+    for name in self.GetAllSlots():
+      if hasattr(self, name):
+        state[name] = getattr(self, name)
+    return state
+
+  def __setstate__(self, state):
+    """Generic unserializer.
+
+    This method just restores from the serialized state the attributes
+    of the current instance.
+
+    @param state: the serialized opcode data
+    @type state:  C{dict}
+
+    """
+    if not isinstance(state, dict):
+      raise ValueError("Invalid data to __setstate__: expected dict, got %s" %
+                       type(state))
+
+    for name in self.GetAllSlots():
+      if name not in state and hasattr(self, name):
+        delattr(self, name)
+
+    for name in state:
+      setattr(self, name, state[name])
+
+  @classmethod
+  def GetAllParams(cls):
+    """Compute list of all parameters for an opcode.
+
+    """
+    slots = []
+    for parent in cls.__mro__:
+      slots.extend(getattr(parent, "OP_PARAMS", []))
+    return slots
+
+  def Validate(self, set_defaults): # pylint: disable=W0221
+    """Validate opcode parameters, optionally setting default values.
+
+    @type set_defaults: bool
+    @param set_defaults: Whether to set default values
+    @raise errors.OpPrereqError: When a parameter value doesn't match
+                                 requirements
+
+    """
+    for (attr_name, default, test, _) in self.GetAllParams():
+      assert callable(test)
+
+      if hasattr(self, attr_name):
+        attr_val = getattr(self, attr_name)
+      else:
+        attr_val = copy.deepcopy(default)
+
+      if test(attr_val):
+        if set_defaults:
+          setattr(self, attr_name, attr_val)
+      elif ht.TInt(attr_val) and test(float(attr_val)):
+        if set_defaults:
+          setattr(self, attr_name, float(attr_val))
+      else:
+        logging.error("OpCode %s, parameter %s, has invalid type %s/value"
+                      " '%s' expecting type %s",
+                      self.OP_ID, attr_name, type(attr_val), attr_val, test)
+
+        if attr_val is None:
+          logging.error("OpCode %s, parameter %s, has default value None which"
+                        " is does not check against the parameter's type: this"
+                        " means this parameter is required but no value was"
+                        " given",
+                        self.OP_ID, attr_name)
+
+        raise errors.OpPrereqError("Parameter '%s.%s' fails validation" %
+                                   (self.OP_ID, attr_name),
+                                   errors.ECODE_INVAL)
+
+
+def BuildJobDepCheck(relative):