Commit 49598ac3 authored by Hrvoje Ribicic's avatar Hrvoje Ribicic

Merge branch 'stable-2.12' into stable-2.13

* stable-2.12
  Expand orphan volume test
  Restrict Ganeti's orphan volume checks to the single VG
  Modify UDS server startup to set permissions for sockets
  Add wheezy chroot files to gitignore file
  Makefile.am: Don't use -dynamic-too for .hpc_o files
  Makefile.am: Don't use dots in -osuf
  Fix compiler invocation for GHC >= 7.8
  Makefile.am: Fix wrong -dep-suffix for GHC 7.8
  Only upgrade configs not upgraded
  Only unlock config if we did lock it
  Mention preferred DRBD module settings when using Xen
  Avoid assertIn
  Test presence of public and private parameters
  Put private parameters into the environment
  Always close pipe on job forking
  Clean up pipes early on failed forks

Conflicts:
	test/py/ganeti.backend_unittest.py

Resolution:
        Merge all test changes.
Signed-off-by: default avatarHrvoje Ribicic <riba@google.com>
Reviewed-by: default avatarKlaus Aehlig <aehlig@google.com>
parents 1d0abda4 ebeed788
......@@ -42,6 +42,8 @@
/configure
/devel/squeeze-amd64.tar.gz
/devel/squeeze-amd64.conf
/devel/wheezy-amd64.tar.gz
/devel/wheezy-amd64.conf
/epydoc.conf
/ganeti
/stamp-srclinks
......
......@@ -767,6 +767,28 @@ endif
HTEST_SUFFIX = hpc
HPROF_SUFFIX = prof
DEP_SUFFIXES =
if GHC_LE_76
DEP_SUFFIXES += -dep-suffix $(HPROF_SUFFIX) -dep-suffix $(HTEST_SUFFIX)
else
# GHC >= 7.8 stopped putting underscores into -dep-suffix by itself
# (https://ghc.haskell.org/trac/ghc/ticket/9749) so we have to put them.
# It also needs -dep-suffix "" for the .o file.
DEP_SUFFIXES += -dep-suffix $(HPROF_SUFFIX)_ -dep-suffix $(HTEST_SUFFIX)_ \
-dep-suffix ""
endif
# GHC > 7.6 needs -dynamic-too when using Template Haskell since its
# ghci is switched to loading dynamic libraries by default.
# It must only be used in non-profiling GHC invocations.
# We also don't use it in compilations that use HTEST_SUFFIX (which are
# compiled with -fhpc) because HPC coverage doesn't interact well with
# GHCI shared lib loading (https://ghc.haskell.org/trac/ghc/ticket/9762).
HFLAGS_DYNAMIC =
if !GHC_LE_76
HFLAGS_DYNAMIC += -dynamic-too
endif
if HPROFILE
HPROFFLAGS = -prof -fprof-auto-top -osuf $(HPROF_SUFFIX)_o \
-hisuf $(HPROF_SUFFIX)_hi -rtsopts
......@@ -779,8 +801,8 @@ HFLAGS += -DTEST
endif
HTEST_FLAGS = $(HFLAGS) -fhpc -itest/hs \
-osuf .$(HTEST_SUFFIX)_o \
-hisuf .$(HTEST_SUFFIX)_hi
-osuf $(HTEST_SUFFIX)_o \
-hisuf $(HTEST_SUFFIX)_hi
# extra flags that can be overriden on the command line (e.g. -Wwarn, etc.)
HEXTRA =
......@@ -1267,8 +1289,8 @@ HS_MAKEFILE_GHC_SRCS += $(HS_TEST_PROGS:%=%.hs)
endif
Makefile.ghc: $(HS_MAKEFILE_GHC_SRCS) Makefile hs-pkg-versions \
| $(built_base_sources) $(HS_BUILT_SRCS)
$(GHC) -M -dep-makefile $@ -dep-suffix $(HPROF_SUFFIX) \
-dep-suffix $(HTEST_SUFFIX) $(HFLAGS) -itest/hs \
$(GHC) -M -dep-makefile $@ $(DEP_SUFFIXES) $(HFLAGS) $(HFLAGS_DYNAMIC) \
-itest/hs \
$(shell cat hs-pkg-versions) \
$(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(HS_MAKEFILE_GHC_SRCS)
# Since ghc -M does not generate dependency line for object files, dependencies
......@@ -1285,9 +1307,20 @@ Makefile.ghc: $(HS_MAKEFILE_GHC_SRCS) Makefile hs-pkg-versions \
@include_makefile_ghc@
# Like the %.o rule, but allows access to the test/hs directory.
# This uses HFLAGS instead of HTEST_FLAGS because it's only for generating
# object files (.o for GHC <= 7.6, .o/.so for newer GHCs) that are loaded
# in GHCI when evaluating TH. The actual test-with-coverage .hpc_o files
# are created in the `%.$(HTEST_SUFFIX)_o` rule.
test/hs/%.o: hs-pkg-versions
@echo '[GHC|test]: $@ <- $^'
@$(GHC) -c $(HFLAGS) $(HFLAGS_DYNAMIC) -itest/hs \
$(shell cat hs-pkg-versions) \
$(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(@:%.o=%.hs)
%.o: hs-pkg-versions
@echo '[GHC]: $@ <- $^'
@$(GHC) -c $(HFLAGS) \
@$(GHC) -c $(HFLAGS) $(HFLAGS_NOPROF) \
$(shell cat hs-pkg-versions) \
$(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(@:%.o=%.hs)
......@@ -1304,7 +1337,11 @@ if HPROFILE
$(@:%.$(HPROF_SUFFIX)_o=%.hs)
endif
%.$(HTEST_SUFFIX)_o: hs-pkg-versions
# We depend on the non-test .o file here because we need the corresponding .so
# file for GHC > 7.6 ghci dynamic loading for TH, and creating the .o file
# will create the .so file since we use -dynamic-too (using the `test/hs/%.o`
# rule).
%.$(HTEST_SUFFIX)_o: %.o hs-pkg-versions
@echo '[GHC|test]: $@ <- $^'
@$(GHC) -c $(HTEST_FLAGS) \
$(shell cat hs-pkg-versions) \
......@@ -1316,11 +1353,14 @@ endif
if HPROFILE
$(HS_SRC_PROGS): %: %.$(HPROF_SUFFIX)_o | stamp-directories
@echo '[GHC-link]: $@'
$(GHC) $(HFLAGS) $(HPROFFLAGS) \
$(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) --make $(@:%=%.hs)
else
$(HS_SRC_PROGS): %: %.o hs-pkg-versions | stamp-directories
endif
@echo '[GHC-link]: $@'
$(GHC) $(HFLAGS) \
$(GHC) $(HFLAGS) $(HFLAGS_DYNAMIC) \
$(shell cat hs-pkg-versions) \
$(HPROFFLAGS) \
$(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) --make $(@:%=%.hs)
......
......@@ -629,6 +629,10 @@ if test -z "$GHC"; then
AC_MSG_FAILURE([ghc not found, compilation will not possible])
fi
# Note: Character classes ([...]) need to be double quoted due to autoconf
# using m4
AM_CONDITIONAL([GHC_LE_76], [$GHC --numeric-version | grep -q '^7\.[[0-6]]\.'])
AC_MSG_CHECKING([checking for extra GHC flags])
GHC_BYVERSION_FLAGS=
# check for GHC supported flags that vary accross versions
......
......@@ -251,7 +251,11 @@ your instances to DRBD to take advantage of the new features.
Supported DRBD versions: 8.0-8.4. It's recommended to have at least
version 8.0.12. Note that for version 8.2 and newer it is needed to pass
the ``usermode_helper=/bin/true`` parameter to the module, either by
configuring ``/etc/modules`` or when inserting it manually.
configuring ``/etc/modules`` or when inserting it manually. When using
Xen and DRBD 8.3.2 or higher, it is recommended_ to use the
``disable_sendpage=1`` setting as well.
.. _recommended: https://drbd.linbit.com/users-guide/s-xen-drbd-mod-params.html
Now the bad news: unless your distribution already provides it
installing DRBD might involve recompiling your kernel or anyway fiddling
......
......@@ -1238,7 +1238,7 @@ def VerifyNode(what, cluster_name, all_hvparams, node_groups, groups_cfg):
if constants.NV_LVLIST in what and vm_capable:
try:
val = GetVolumeList([what[constants.NV_LVLIST]])
val = GetVolumeList(utils.ListVolumeGroups().keys())
except RPCFail, err:
val = str(err)
result[constants.NV_LVLIST] = val
......@@ -4075,7 +4075,8 @@ def OSEnvironment(instance, inst_os, debug=0):
cannot be found
"""
result = OSCoreEnv(instance.os, inst_os, instance.osparams, debug=debug)
result = OSCoreEnv(instance.os, inst_os, objects.FillDict(instance.osparams,
instance.osparams_private.Unprivate()), debug=debug)
for attr in ["name", "os", "uuid", "ctime", "mtime", "primary_node"]:
result["INSTANCE_%s" % attr.upper()] = str(getattr(instance, attr))
......
......@@ -2716,12 +2716,15 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
"instance lives on non-vm_capable node %s",
self.cfg.GetNodeName(node_uuid))
def _VerifyOrphanVolumes(self, node_vol_should, node_image, reserved):
def _VerifyOrphanVolumes(self, vg_name, node_vol_should, node_image,
reserved):
"""Verify if there are any unknown volumes in the cluster.
The .os, .swap and backup volumes are ignored. All other volumes are
reported as unknown.
@type vg_name: string
@param vg_name: the name of the Ganeti-administered volume group
@type reserved: L{ganeti.utils.FieldSet}
@param reserved: a FieldSet of reserved volume names
......@@ -2732,6 +2735,10 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
# skip non-healthy nodes
continue
for volume in n_img.volumes:
# skip volumes not belonging to the ganeti-administered volume group
if volume.split('/')[0] != vg_name:
continue
test = ((node_uuid not in node_vol_should or
volume not in node_vol_should[node_uuid]) and
not reserved.Matches(volume))
......@@ -3934,7 +3941,7 @@ class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors):
self.cfg.GetInstanceLVsByNode(instance.uuid, lvmap=node_vol_should)
break
self._VerifyOrphanVolumes(node_vol_should, node_image, reserved)
self._VerifyOrphanVolumes(vg_name, node_vol_should, node_image, reserved)
if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks:
feedback_fn("* Verifying N+1 Memory redundancy")
......
......@@ -2901,13 +2901,10 @@ class ConfigWriter(object):
try:
if dict_data is not None:
self._SetConfigData(objects.ConfigData.FromDict(dict_data))
self._UpgradeConfig()
except Exception, err:
raise errors.ConfigurationError(err)
# Transitional fix until ConfigWriter is completely rewritten into
# Haskell
self._UpgradeConfig()
def _CloseConfig(self, save):
"""Release resources relating the config data.
......@@ -2921,7 +2918,7 @@ class ConfigWriter(object):
logging.critical("Can't write the configuration: %s", str(err))
raise
finally:
if not self._offline:
if not self._offline and not self._lock_current_shared:
try:
self._wconfd.UnlockConfig(self._GetWConfdContext())
except AttributeError:
......
......@@ -123,6 +123,7 @@ import Ganeti.Path
import Ganeti.Query.Exec as Exec
import Ganeti.Rpc (executeRpcCall, ERpcError, logRpcErrors,
RpcCallJobqueueUpdate(..), RpcCallJobqueueRename(..))
import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..))
import Ganeti.Types
import Ganeti.Utils
import Ganeti.Utils.Atomic
......@@ -646,8 +647,8 @@ notifyJob pid = runResultT $ do
-- | Permissions for the archive directories.
queueDirPermissions :: FilePermissions
queueDirPermissions = FilePermissions { fpOwner = Just C.masterdUser
, fpGroup = Just C.daemonsGroup
queueDirPermissions = FilePermissions { fpOwner = Just GanetiMasterd
, fpGroup = Just $ ExtraGroup DaemonsGroup
, fpPermissions = 0o0750
}
......
......@@ -77,11 +77,12 @@ import Ganeti.Objects
import Ganeti.OpParams (pTagsObject)
import Ganeti.OpCodes
import qualified Ganeti.Query.Language as Qlang
import Ganeti.Runtime (GanetiDaemon(..))
import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..))
import Ganeti.THH
import Ganeti.THH.Field
import Ganeti.THH.Types (getOneTuple)
import Ganeti.Types
import Ganeti.Utils
-- | Currently supported Luxi operations and JSON serialization.
......@@ -200,7 +201,13 @@ $(genStrOfOp ''LuxiOp "strOfOp")
luxiConnectConfig :: ServerConfig
luxiConnectConfig = ServerConfig GanetiLuxid
luxiConnectConfig = ServerConfig
-- The rapi daemon talks to the luxi one, and for this
-- purpose we need group rw permissions.
FilePermissions { fpOwner = Just GanetiLuxid
, fpGroup = Just $ ExtraGroup DaemonsGroup
, fpPermissions = 0o0660
}
ConnectConfig { recvTmo = luxiDefRwto
, sendTmo = luxiDefRwto
}
......
......@@ -43,9 +43,10 @@ import System.IO.Error (isEOFError)
import Ganeti.Path as Path
import Ganeti.Daemon (DaemonOptions, cleanupSocket, describeError)
import qualified Ganeti.Logging as Logging
import Ganeti.Runtime (GanetiDaemon(..))
import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..))
import Ganeti.UDSServer (Client, ConnectConfig(..), Server, ServerConfig(..))
import qualified Ganeti.UDSServer as UDSServer
import Ganeti.Utils (FilePermissions(..))
import Ganeti.Metad.Config as Config
import Ganeti.Metad.Types (InstanceParams)
......@@ -100,4 +101,15 @@ start _ config = do
(acceptClients config server)
(UDSServer.closeServer server)
where
metadConfig = ServerConfig GanetiMetad $ ConnectConfig 60 60
metadConfig =
ServerConfig
-- The permission 0600 is completely acceptable because only the node
-- daemon talks to the metadata daemon, and the node daemon runs as
-- root.
FilePermissions { fpOwner = Just GanetiMetad
, fpGroup = Just $ ExtraGroup DaemonsGroup
, fpPermissions = 0o0600
}
ConnectConfig { recvTmo = 60
, sendTmo = 60
}
......@@ -62,6 +62,7 @@ module Ganeti.Query.Exec
import Control.Concurrent (rtsSupportsBoundThreads)
import Control.Concurrent.Lifted (threadDelay)
import Control.Exception (finally)
import Control.Monad
import Control.Monad.Error
import Data.Functor
......@@ -194,8 +195,9 @@ runJobProcess jid s = withErrorLogAt CRITICAL (show jid) $
forkWithPipe :: ConnectConfig -> (Client -> IO ()) -> IO (ProcessID, Client)
forkWithPipe conf childAction = do
(master, child) <- pipeClient conf
pid <- forkProcess (closeClient master >> childAction child)
closeClient child
pid <- finally
(forkProcess (closeClient master >> childAction child))
$ closeClient child
return (pid, master)
-- | Forks the job process and starts processing of the given job.
......@@ -273,4 +275,6 @@ forkJobProcess jid luxiLivelock update = do
_ <- recv "Waiting for the job to ask for the lock file name"
send "Writing the lock file name to the client" lockfile
liftIO $ closeClient master
return (lockfile, pid)
......@@ -98,7 +98,6 @@ import Ganeti.BasicTypes
import Ganeti.Errors (GanetiException(..), ErrorResult)
import Ganeti.JSON
import Ganeti.Logging
import Ganeti.Runtime (GanetiDaemon(..), MiscGroup(..), GanetiGroup(..))
import Ganeti.THH
import Ganeti.Utils
import Ganeti.Constants (privateParametersBlacklist)
......@@ -143,7 +142,7 @@ $(genStrOfKey ''MsgKeys "strOfKey")
-- Information required for creating a server connection.
data ServerConfig = ServerConfig
{ connDaemon :: GanetiDaemon
{ connPermissions :: FilePermissions
, connConfig :: ConnectConfig
}
......@@ -226,8 +225,10 @@ connectClient conf tmo path = do
connectServer :: ServerConfig -> Bool -> FilePath -> IO Server
connectServer sconf setOwner path = do
s <- openServerSocket path
when setOwner . setOwnerAndGroupFromNames path (connDaemon sconf) $
ExtraGroup DaemonsGroup
when setOwner $ do
res <- ensurePermissions path (connPermissions sconf)
exitIfBad "Error - could not set socket properties" res
S.listen s 5 -- 5 is the max backlog
return Server { sSocket = s, sPath = path, serverConfig = connConfig sconf }
......
......@@ -130,7 +130,6 @@ import System.IO
import System.Exit
import System.Posix.Files
import System.Posix.IO
import System.Posix.User
import System.Time
-- * Debug functions
......@@ -732,8 +731,8 @@ watchFile fpath timeout old = watchFileBy fpath timeout (/= old)
-- directories and files. All parameters are optional, with nothing
-- meaning that the default value should be left untouched.
data FilePermissions = FilePermissions { fpOwner :: Maybe String
, fpGroup :: Maybe String
data FilePermissions = FilePermissions { fpOwner :: Maybe GanetiDaemon
, fpGroup :: Maybe GanetiGroup
, fpPermissions :: FileMode
}
......@@ -741,22 +740,29 @@ data FilePermissions = FilePermissions { fpOwner :: Maybe String
-- possibly ownerships, as required.
ensurePermissions :: FilePath -> FilePermissions -> IO (Result ())
ensurePermissions fpath perms = do
-- Fetch the list of entities
runtimeEnts <- runResultT getEnts
ents <- exitIfBad "Can't determine user/group ids" runtimeEnts
-- Get the existing file properties
eitherFileStatus <- try $ getFileStatus fpath
:: IO (Either IOError FileStatus)
-- And see if any modifications are needed
(flip $ either (return . Bad . show)) eitherFileStatus $ \fstat -> do
ownertry <- case fpOwner perms of
Nothing -> return $ Right ()
Just owner -> try $ do
ownerid <- userID `liftM` getUserEntryForName owner
let ownerid = reUserToUid ents M.! owner
unless (ownerid == fileOwner fstat) $ do
logDebug $ "Changing owner of " ++ fpath ++ " to " ++ owner
logDebug $ "Changing owner of " ++ fpath ++ " to " ++ show owner
setOwnerAndGroup fpath ownerid (-1)
grouptry <- case fpGroup perms of
Nothing -> return $ Right ()
Just grp -> try $ do
groupid <- groupID `liftM` getGroupEntryForName grp
let groupid = reGroupToGid ents M.! grp
unless (groupid == fileGroup fstat) $ do
logDebug $ "Changing group of " ++ fpath ++ " to " ++ grp
logDebug $ "Changing group of " ++ fpath ++ " to " ++ show grp
setOwnerAndGroup fpath (-1) groupid
let fp = fpPermissions perms
permtry <- if fileMode fstat == fp
......
......@@ -55,6 +55,7 @@ import Ganeti.THH.RPC
import Ganeti.UDSServer
import Ganeti.Errors (formatError)
import Ganeti.Runtime
import Ganeti.Utils
import Ganeti.Utils.Livelock (mkLivelockFile)
import Ganeti.WConfd.ConfigState
import Ganeti.WConfd.ConfigVerify
......@@ -109,7 +110,17 @@ prepMain _ _ = do
return (s, dh)
serverConfig :: ServerConfig
serverConfig = ServerConfig GanetiWConfd $ ConnectConfig 60 60
serverConfig = ServerConfig
-- All the daemons that need to talk to WConfd should be
-- running as the same user - the former master daemon user.
FilePermissions { fpOwner = Just GanetiWConfd
, fpGroup = Just $ ExtraGroup DaemonsGroup
, fpPermissions = 0o0600
}
ConnectConfig { recvTmo = 60
, sendTmo = 60
}
-- | Main function.
main :: MainFn () PrepResult
......
......@@ -1714,19 +1714,27 @@ class TestLUClusterVerifyGroupVerifyOrphanVolumes(
@withLockedLU
def testOrphanedVolume(self, lu):
master_img = cluster.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid)
master_img.volumes = ["mock_vg/disk_0", "mock_vg/disk_1", "mock_vg/disk_2"]
master_img.volumes = [
"mock_vg/disk_0", # Required, present, no error
"mock_vg/disk_1", # Unknown, present, orphan
"mock_vg/disk_2", # Reserved, present, no error
"other_vg/disk_0", # Required, present, no error
"other_vg/disk_1", # Unknown, present, no error
]
node_imgs = {
self.master_uuid: master_img
}
node_vol_should = {
self.master_uuid: ["mock_vg/disk_0"]
self.master_uuid: ["mock_vg/disk_0", "other_vg/disk_0", "other_vg/disk_1"]
}
lu._VerifyOrphanVolumes(node_vol_should, node_imgs,
lu._VerifyOrphanVolumes("mock_vg", node_vol_should, node_imgs,
utils.FieldSet("mock_vg/disk_2"))
self.mcpu.assertLogContainsRegex("volume mock_vg/disk_1 is unknown")
self.mcpu.assertLogDoesNotContainRegex("volume mock_vg/disk_0 is unknown")
self.mcpu.assertLogDoesNotContainRegex("volume mock_vg/disk_2 is unknown")
self.mcpu.assertLogDoesNotContainRegex("volume other_vg/disk_0 is unknown")
self.mcpu.assertLogDoesNotContainRegex("volume other_vg/disk_1 is unknown")
class TestLUClusterVerifyGroupVerifyNPlusOneMemory(
......
......@@ -46,8 +46,10 @@ from ganeti import hypervisor
from ganeti import netutils
from ganeti import objects
from ganeti import pathutils
from ganeti import serializer
from ganeti import ssh
from ganeti import utils
from cmdlib.testsupport.config_mock import ConfigMock
class TestX509Certificates(unittest.TestCase):
......@@ -1756,5 +1758,38 @@ class TestVerifySshSetup(testutils.GanetiTestCase):
self.assertTrue(self._NODE3_UUID in result[0])
class TestOSEnvironment(unittest.TestCase):
"""Ensure the presence of public and private parameters.
They have to be present inside os environment variables.
"""
def _CreateEnv(self):
"""Create and return an environment."""
config_mock = ConfigMock()
inst = config_mock.AddNewInstance(
osparams={"public_param": "public_info"},
osparams_private=serializer.PrivateDict({"private_param":
"private_info",
"another_private_param":
"more_privacy"}),
nics = [])
inst.disks_info = ""
inst.secondary_nodes = []
return backend.OSEnvironment(inst, config_mock.CreateOs())
def testParamPresence(self):
env = self._CreateEnv()
env_keys = env.keys()
self.assertTrue("OSP_PUBLIC_PARAM" in env)
self.assertTrue("OSP_PRIVATE_PARAM" in env)
self.assertTrue("OSP_ANOTHER_PRIVATE_PARAM" in env)
self.assertEqual("public_info", env["OSP_PUBLIC_PARAM"])
self.assertEqual("private_info", env["OSP_PRIVATE_PARAM"])
self.assertEqual("more_privacy", env["OSP_ANOTHER_PRIVATE_PARAM"])
if __name__ == "__main__":
testutils.GanetiTestProgram()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment