Add the compression tools parameter

This patch makes the myriad of changes necessary for the compression
tool parameter to be added. The filtering of compression tools for
suspicious entries has been added for this exact purpose.
Signed-off-by: Hrvoje Ribicic <riba@google.com>
Reviewed-by: Thomas Thrainer <thomasth@google.com>

lib/bootstrap.py

@@ -553,7 +553,7 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
                 primary_ip_version=None, ipolicy=None,
                 prealloc_wipe_disks=False, use_external_mip_script=False,
                 hv_state=None, disk_state=None, enabled_disk_templates=None,
-                zeroing_image=None):
+                zeroing_image=None, compression_tools=None):
   """Initialise the cluster.
 
   @type candidate_pool_size: int
@@ -780,6 +780,9 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
 
   now = time.time()
 
+  if compression_tools is not None:
+    cluster.CheckCompressionTools(compression_tools)
+
   # init of cluster config file
   cluster_config = objects.Cluster(
     serial_no=1,
@@ -822,7 +825,8 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
     candidate_certs=candidate_certs,
     osparams={},
     osparams_private_cluster={},
-    zeroing_image=zeroing_image
+    zeroing_image=zeroing_image,
+    compression_tools=compression_tools
     )
   master_node_config = objects.Node(name=hostname.name,
                                     primary_ip=hostname.ip,

lib/cli.py

@@ -71,6 +71,7 @@ __all__ = [
   "CLUSTER_DOMAIN_SECRET_OPT",
   "CONFIRM_OPT",
   "CP_SIZE_OPT",
+  "COMPRESSION_TOOLS_OPT",
   "DEBUG_OPT",
   "DEBUG_SIMERR_OPT",
   "DISKIDX_OPT",
@@ -1315,6 +1316,12 @@ INSTANCE_COMMUNICATION_NETWORK_OPT = \
                type="string",
                help="Set the network name for instance communication")
 
+COMPRESSION_TOOLS_OPT = \
+    cli_option("--compression-tools",
+               dest="compression_tools", type="string", default=None,
+               help="Comma-separated list of compression tools which are"
+                    " allowed to be used by Ganeti in various operations")
+
 VG_NAME_OPT = cli_option("--vg-name", dest="vg_name",
                          help=("Enables LVM and specifies the volume group"
                                " name (cluster-wide) for disk allocation"

lib/client/gnt_cluster.py

@@ -270,6 +270,8 @@ def InitCluster(opts, args):
   else:
     zeroing_image = ""
 
+  compression_tools = _GetCompressionTools(opts)
+
   default_ialloc_params = opts.default_iallocator_params
   bootstrap.InitCluster(cluster_name=args[0],
                         secondary_ip=opts.secondary_ip,
@@ -301,7 +303,8 @@ def InitCluster(opts, args):
                         hv_state=hv_state,
                         disk_state=disk_state,
                         enabled_disk_templates=enabled_disk_templates,
-                        zeroing_image=zeroing_image
+                        zeroing_image=zeroing_image,
+                        compression_tools=compression_tools
                         )
   op = opcodes.OpClusterPostInit()
   SubmitOpCode(op, opts=opts)
@@ -555,6 +558,7 @@ def ShowClusterConfig(opts, args):
       ("instance communication network",
        result["instance_communication_network"]),
       ("zeroing image", result["zeroing_image"]),
+      ("compression tools", result["compression_tools"]),
       ]),
 
     ("Default node parameters",
@@ -1092,6 +1096,18 @@ def _GetDrbdHelper(opts, enabled_disk_templates):
   return drbd_helper
 
 
+def _GetCompressionTools(opts):
+  """Determine the list of custom compression tools.
+
+  """
+  if opts.compression_tools:
+    return opts.compression_tools.split(",")
+  elif opts.compression_tools is None:
+    return None # To note the parameter was not provided
+  else:
+    return constants.IEC_DEFAULT_TOOLS # Resetting to default
+
+
 def SetClusterParams(opts, args):
   """Modify the cluster.
 
@@ -1133,7 +1149,8 @@ def SetClusterParams(opts, args):
           opts.file_storage_dir is not None or
           opts.instance_communication_network is not None or
           opts.zeroing_image is not None or
-          opts.shared_file_storage_dir is not None):
+          opts.shared_file_storage_dir is not None or
+          opts.compression_tools is not None):
     ToStderr("Please give at least one of the parameters.")
     return 1
 
@@ -1214,6 +1231,8 @@ def SetClusterParams(opts, args):
 
   hv_state = dict(opts.hv_state)
 
+  compression_tools = _GetCompressionTools(opts)
+
   op = opcodes.OpClusterSetParams(
     vg_name=vg_name,
     drbd_helper=drbd_helper,
@@ -1248,6 +1267,7 @@ def SetClusterParams(opts, args):
     instance_communication_network=opts.instance_communication_network,
     zeroing_image=opts.zeroing_image,
     shared_file_storage_dir=opts.shared_file_storage_dir,
+    compression_tools=compression_tools
     )
   return base.GetResult(None, opts, SubmitOrSend(op, opts))
 
@@ -2112,7 +2132,8 @@ commands = {
      PRIMARY_IP_VERSION_OPT, PREALLOC_WIPE_DISKS_OPT, NODE_PARAMS_OPT,
      GLOBAL_SHARED_FILEDIR_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT,
      HV_STATE_OPT, DISK_STATE_OPT, ENABLED_DISK_TEMPLATES_OPT,
-     IPOLICY_STD_SPECS_OPT, GLOBAL_GLUSTER_FILEDIR_OPT, ZEROING_IMAGE_OPT]
+     IPOLICY_STD_SPECS_OPT, GLOBAL_GLUSTER_FILEDIR_OPT, ZEROING_IMAGE_OPT,
+     COMPRESSION_TOOLS_OPT]
      + INSTANCE_POLICY_OPTS + SPLIT_ISPECS_OPTS,
     "[opts...] <cluster_name>", "Initialises a new cluster configuration"),
   "destroy": (
@@ -2197,7 +2218,8 @@ commands = {
      SUBMIT_OPTS +
      [ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT, MODIFY_ETCHOSTS_OPT] +
      INSTANCE_POLICY_OPTS +
-     [GLOBAL_FILEDIR_OPT, GLOBAL_SHARED_FILEDIR_OPT, ZEROING_IMAGE_OPT],
+     [GLOBAL_FILEDIR_OPT, GLOBAL_SHARED_FILEDIR_OPT, ZEROING_IMAGE_OPT,
+      COMPRESSION_TOOLS_OPT],
     "[opts...]",
     "Alters the parameters of the cluster"),
   "renew-crypto": (

lib/cmdlib/cluster.py

@@ -416,6 +416,7 @@ class LUClusterQuery(NoHooksLU):
       "blacklisted_os": cluster.blacklisted_os,
       "enabled_disk_templates": cluster.enabled_disk_templates,
       "instance_communication_network": cluster.instance_communication_network,
+      "compression_tools": cluster.compression_tools,
       }
 
     return result
@@ -767,6 +768,33 @@ def CheckSharedFileStoragePathVsEnabledDiskTemplates(
       constants.DT_SHARED_FILE)
 
 
+def CheckCompressionTools(tools):
+  """Check whether the provided compression tools look like executables.
+
+  @type tools: list of string
+  @param tools: The tools provided as opcode input
+
+  """
+  regex = re.compile('^[-_a-zA-Z0-9]+$')
+  illegal_tools = [t for t in tools if not regex.match(t)]
+
+  if illegal_tools:
+    raise errors.OpPrereqError(
+      "The tools '%s' contain illegal characters: only alphanumeric values,"
+      " dashes, and underscores are allowed" % ", ".join(illegal_tools)
+    )
+
+  if constants.IEC_GZIP not in tools:
+    raise errors.OpPrereqError("For compatibility reasons, the %s utility must"
+                               " be present among the compression tools" %
+                               constants.IEC_GZIP)
+
+  if constants.IEC_NONE in tools:
+    raise errors.OpPrereqError("%s is a reserved value used for no compression,"
+                               " and cannot be used as the name of a tool" %
+                               constants.IEC_NONE)
+
+
 class LUClusterSetParams(LogicalUnit):
   """Change the parameters of the cluster.
 
@@ -1273,6 +1301,9 @@ class LUClusterSetParams(LogicalUnit):
         network = self.cfg.GetNetwork(network_uuid)
         self._CheckInstanceCommunicationNetwork(network, self.LogWarning)
 
+    if self.op.compression_tools:
+      CheckCompressionTools(self.op.compression_tools)
+
   def _BuildOSParams(self, cluster):
     "Calculate the new OS parameters for this operation."
 
@@ -1645,6 +1676,9 @@ class LUClusterSetParams(LogicalUnit):
       result.Warn("Could not re-enable the master ip on the master,"
                   " please restart manually", self.LogWarning)
 
+    if self.op.compression_tools is not None:
+      self.cfg.SetCompressionTools(self.op.compression_tools)
+
     network_name = self.op.instance_communication_network
     if network_name is not None:
       return self._ModifyInstanceCommunicationNetwork(self.cfg,

lib/config.py

@@ -1612,6 +1612,28 @@ class ConfigWriter(object):
     """
     return self._config_data.cluster.zeroing_image
 
+  @_ConfigSync(shared=1)
+  def GetCompressionTools(self):
+    """Get cluster compression tools
+
+    @rtype: list of string
+    @return: a list of tools that are cleared for use in this cluster for the
+             purpose of compressing data
+
+    """
+    return self._ConfigData().cluster.compression_tools
+
+  @_ConfigSync()
+  def SetCompressionTools(self, tools):
+    """Set cluster compression tools
+
+    @type tools: list of string
+    @param tools: a list of tools that are cleared for use in this cluster for
+                  the purpose of compressing data
+
+    """
+    self._ConfigData().cluster.compression_tools = tools
+
   @_ConfigSync()
   def AddNodeGroup(self, group, ec_id, check_uuid=True):
     """Add a node group to the configuration.

lib/objects.py

@@ -1589,7 +1589,8 @@ class Cluster(TaggableObject):
     "candidate_certs",
     "max_running_jobs",
     "instance_communication_network",
-    "zeroing_image"
+    "zeroing_image",
+    "compression_tools",
     ] + _TIMESTAMPS + _UUID
 
   def UpgradeConfig(self):
@@ -1727,6 +1728,9 @@ class Cluster(TaggableObject):
     if self.instance_communication_network is None:
       self.instance_communication_network = ""
 
+    if self.compression_tools is None:
+      self.compression_tools = constants.IEC_DEFAULT_TOOLS
+
   @property
   def primary_hypervisor(self):
     """The first hypervisor is the primary.

src/Ganeti/Constants.hs

@@ -601,6 +601,9 @@ iecNone = "none"
 iecAll :: [String]
 iecAll = [iecGzip, iecGzipFast, iecGzipSlow, iecLzop, iecNone]
 
+iecDefaultTools :: [String]
+iecDefaultTools = [iecGzip, iecGzipFast, iecGzipSlow]
+
 iecCompressionUtilities :: Map String String
 iecCompressionUtilities =
   Map.fromList

src/Ganeti/Objects.hs

@@ -725,6 +725,7 @@ $(buildObject "Cluster" "cluster" $
   , simpleField "max_running_jobs"               [t| Int                    |]
   , simpleField "instance_communication_network" [t| String                 |]
   , simpleField "zeroing_image"                  [t| String                 |]
+  , simpleField "compression_tools"              [t| [String]               |]
  ]
  ++ timeStampFields
  ++ uuidFields

src/Ganeti/OpCodes.hs

@@ -241,6 +241,7 @@ $(genOpCode "OpCode"
      , pClusterGlusterStorageDir
      , pInstanceCommunicationNetwork
      , pZeroingImage
+     , pCompressionTools
      ],
      [])
   , ("OpClusterRedistConf",

src/Ganeti/OpParams.hs

@@ -111,6 +111,7 @@ module Ganeti.OpParams
   , pClusterGlusterStorageDir
   , pInstanceCommunicationNetwork
   , pZeroingImage
+  , pCompressionTools
   , pVgName
   , pEnabledHypervisors
   , pHypervisor
@@ -604,6 +605,12 @@ pZeroingImage :: Field
 pZeroingImage =
   optionalStringField "zeroing_image"
 
+-- | The additional tools that can be used to compress data in transit
+pCompressionTools :: Field
+pCompressionTools =
+  withDoc "List of enabled compression tools" . optionalField $
+  simpleField "compression_tools" [t| [NonEmptyString] |]
+
 -- | Volume group name.
 pVgName :: Field
 pVgName =

src/Ganeti/Query/Server.hs

@@ -173,6 +173,8 @@ handleCall _ _ cdata QueryClusterInfo =
             , ("instance_communication_network",
                showJSON (clusterInstanceCommunicationNetwork cluster))
             , ("zeroing_image", showJSON $ clusterZeroingImage cluster)
+            , ("compression_tools",
+               showJSON $ clusterCompressionTools cluster)
             ]
 
   in case master of

test/hs/Test/Ganeti/OpCodes.hs

@@ -208,6 +208,7 @@ instance Arbitrary OpCodes.OpCode where
           <*> genMaybe genName             -- gluster_file_storage_dir
           <*> arbitrary                    -- instance_communication_network
           <*> arbitrary                    -- zeroing_image
+          <*> arbitrary                    -- compression_tools
       "OP_CLUSTER_REDIST_CONF" -> pure OpCodes.OpClusterRedistConf
       "OP_CLUSTER_ACTIVATE_MASTER_IP" ->
         pure OpCodes.OpClusterActivateMasterIp

test/py/cfgupgrade_unittest.py

@@ -50,6 +50,7 @@ def GetMinimalConfig():
       "candidate_certs": {},
       "instance_communication_network": "",
       "zeroing_image": "",
+      "compression_tools": constants.IEC_DEFAULT_TOOLS,
     },
     "instances": {},
     "disks": {},

tools/cfgupgrade

@@ -156,6 +156,8 @@ def UpgradeCluster(config_data):
     cluster.get("instance_communication_network", "")
   cluster["zeroing_image"] = \
     cluster.get("zeroing_image", "")
+  cluster["compression_tools"] = \
+    cluster.get("compression_tools", constants.IEC_DEFAULT_TOOLS)
 
 
 def UpgradeGroups(config_data):
@@ -455,6 +457,9 @@ def DowngradeCluster(config_data):
   if "zeroing_image" in cluster:
     del cluster["zeroing_image"]
 
+  if "compression_tools" in cluster:
+    del cluster["compression_tools"]
+
 
 def DowngradeGroups(config_data):
   for group in config_data["nodegroups"].values():