Commit 4fd029cf authored by Michael Hanselmann's avatar Michael Hanselmann

utils: Split algorithms into separate file

Signed-off-by: default avatarMichael Hanselmann <hansmi@google.com>
Reviewed-by: default avatarIustin Pop <iustin@google.com>
parent c047a981
......@@ -212,7 +212,8 @@ server_PYTHON = \
lib/server/rapi.py
utils_PYTHON = \
lib/utils/__init__.py
lib/utils/__init__.py \
lib/utils/algo.py
docrst = \
doc/admin.rst \
......@@ -478,8 +479,9 @@ python_tests = \
test/ganeti.serializer_unittest.py \
test/ganeti.ssh_unittest.py \
test/ganeti.uidpool_unittest.py \
test/ganeti.utils_unittest.py \
test/ganeti.utils.algo_unittest.py \
test/ganeti.utils_mlockall_unittest.py \
test/ganeti.utils_unittest.py \
test/ganeti.workerpool_unittest.py \
test/cfgupgrade_unittest.py \
test/docs_unittest.py \
......
#
#
# Copyright (C) 2006, 2007, 2010 Google Inc.
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -62,6 +62,7 @@ from ganeti import errors
from ganeti import constants
from ganeti import compat
from ganeti.utils.algo import * # pylint: disable-msg=W0401
_locksheld = []
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
......@@ -109,9 +110,6 @@ _PARSEUNIT_REGEX = re.compile(r"^([.\d]+)\s*([a-zA-Z]+)?$")
#: ASN1 time regexp
_ASN1_TIME_REGEX = re.compile(r"^(\d+)([-+]\d\d)(\d\d)$")
_SORTER_RE = re.compile("^%s(.*)$" % (8 * "(\D+|\d+)?"))
_SORTER_DIGIT = re.compile("^\d+$")
class RunResult(object):
"""Holds the result of running external programs.
......@@ -1282,52 +1280,6 @@ def BridgeExists(bridge):
return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
def _NiceSortTryInt(val):
"""Attempts to convert a string to an integer.
"""
if val and _SORTER_DIGIT.match(val):
return int(val)
else:
return val
def _NiceSortKey(value):
"""Extract key for sorting.
"""
return [_NiceSortTryInt(grp)
for grp in _SORTER_RE.match(value).groups()]
def NiceSort(values, key=None):
"""Sort a list of strings based on digit and non-digit groupings.
Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
will sort the list in the logical order C{['a1', 'a2', 'a10',
'a11']}.
The sort algorithm breaks each name in groups of either only-digits
or no-digits. Only the first eight such groups are considered, and
after that we just use what's left of the string.
@type values: list
@param values: the names to be sorted
@type key: callable or None
@param key: function of one argument to extract a comparison key from each
list element, must return string
@rtype: list
@return: a copy of the name list sorted with our algorithm
"""
if key is None:
keyfunc = _NiceSortKey
else:
keyfunc = lambda value: _NiceSortKey(key(value))
return sorted(values, key=keyfunc)
def TryConvert(fn, val):
"""Try to convert a value ignoring errors.
......@@ -2193,44 +2145,6 @@ def WaitForFdCondition(fdobj, event, timeout):
return result
def UniqueSequence(seq):
"""Returns a list with unique elements.
Element order is preserved.
@type seq: sequence
@param seq: the sequence with the source elements
@rtype: list
@return: list of unique elements from seq
"""
seen = set()
return [i for i in seq if i not in seen and not seen.add(i)]
def FindDuplicates(seq):
"""Identifies duplicates in a list.
Does not preserve element order.
@type seq: sequence
@param seq: Sequence with source elements
@rtype: list
@return: List of duplicate elements from seq
"""
dup = set()
seen = set()
for item in seq:
if item in seen:
dup.add(item)
else:
seen.add(item)
return list(dup)
def NormalizeAndValidateMac(mac):
"""Normalizes and check if a MAC address is valid.
......
#
#
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Utility functions with algorithms.
"""
import re
_SORTER_RE = re.compile("^%s(.*)$" % (8 * "(\D+|\d+)?"))
_SORTER_DIGIT = re.compile("^\d+$")
def UniqueSequence(seq):
"""Returns a list with unique elements.
Element order is preserved.
@type seq: sequence
@param seq: the sequence with the source elements
@rtype: list
@return: list of unique elements from seq
"""
seen = set()
return [i for i in seq if i not in seen and not seen.add(i)]
def FindDuplicates(seq):
"""Identifies duplicates in a list.
Does not preserve element order.
@type seq: sequence
@param seq: Sequence with source elements
@rtype: list
@return: List of duplicate elements from seq
"""
dup = set()
seen = set()
for item in seq:
if item in seen:
dup.add(item)
else:
seen.add(item)
return list(dup)
def _NiceSortTryInt(val):
"""Attempts to convert a string to an integer.
"""
if val and _SORTER_DIGIT.match(val):
return int(val)
else:
return val
def _NiceSortKey(value):
"""Extract key for sorting.
"""
return [_NiceSortTryInt(grp)
for grp in _SORTER_RE.match(value).groups()]
def NiceSort(values, key=None):
"""Sort a list of strings based on digit and non-digit groupings.
Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
will sort the list in the logical order C{['a1', 'a2', 'a10',
'a11']}.
The sort algorithm breaks each name in groups of either only-digits
or no-digits. Only the first eight such groups are considered, and
after that we just use what's left of the string.
@type values: list
@param values: the names to be sorted
@type key: callable or None
@param key: function of one argument to extract a comparison key from each
list element, must return string
@rtype: list
@return: a copy of the name list sorted with our algorithm
"""
if key is None:
keyfunc = _NiceSortKey
else:
keyfunc = lambda value: _NiceSortKey(key(value))
return sorted(values, key=keyfunc)
#!/usr/bin/python
#
# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
"""Script for testing ganeti.utils.algo"""
import unittest
import random
import operator
from ganeti import constants
from ganeti.utils import algo
import testutils
class TestUniqueSequence(unittest.TestCase):
"""Test case for UniqueSequence"""
def _test(self, input, expected):
self.assertEqual(algo.UniqueSequence(input), expected)
def runTest(self):
# Ordered input
self._test([1, 2, 3], [1, 2, 3])
self._test([1, 1, 2, 2, 3, 3], [1, 2, 3])
self._test([1, 2, 2, 3], [1, 2, 3])
self._test([1, 2, 3, 3], [1, 2, 3])
# Unordered input
self._test([1, 2, 3, 1, 2, 3], [1, 2, 3])
self._test([1, 1, 2, 3, 3, 1, 2], [1, 2, 3])
# Strings
self._test(["a", "a"], ["a"])
self._test(["a", "b"], ["a", "b"])
self._test(["a", "b", "a"], ["a", "b"])
class TestFindDuplicates(unittest.TestCase):
"""Test case for FindDuplicates"""
def _Test(self, seq, expected):
result = algo.FindDuplicates(seq)
self.assertEqual(result, algo.UniqueSequence(result))
self.assertEqual(set(result), set(expected))
def test(self):
self._Test([], [])
self._Test([1, 2, 3], [])
self._Test([9, 8, 8, 0, 5, 1, 7, 0, 6, 7], [8, 0, 7])
for exp in [[1, 2, 3], [3, 2, 1]]:
self._Test([1, 1, 2, 2, 3, 3], exp)
self._Test(["A", "a", "B"], [])
self._Test(["a", "A", "a", "B"], ["a"])
self._Test("Hello World out there!", ["e", " ", "o", "r", "t", "l"])
self._Test(self._Gen(False), [])
self._Test(self._Gen(True), range(1, 10))
@staticmethod
def _Gen(dup):
for i in range(10):
yield i
if dup:
for _ in range(i):
yield i
class TestNiceSort(unittest.TestCase):
def test(self):
self.assertEqual(algo.NiceSort([]), [])
self.assertEqual(algo.NiceSort(["foo"]), ["foo"])
self.assertEqual(algo.NiceSort(["bar", ""]), ["", "bar"])
self.assertEqual(algo.NiceSort([",", "."]), [",", "."])
self.assertEqual(algo.NiceSort(["0.1", "0.2"]), ["0.1", "0.2"])
self.assertEqual(algo.NiceSort(["0;099", "0,099", "0.1", "0.2"]),
["0,099", "0.1", "0.2", "0;099"])
data = ["a0", "a1", "a99", "a20", "a2", "b10", "b70", "b00", "0000"]
self.assertEqual(algo.NiceSort(data),
["0000", "a0", "a1", "a2", "a20", "a99",
"b00", "b10", "b70"])
data = ["a0-0", "a1-0", "a99-10", "a20-3", "a0-4", "a99-3", "a09-2",
"Z", "a9-1", "A", "b"]
self.assertEqual(algo.NiceSort(data),
["A", "Z", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b"])
self.assertEqual(algo.NiceSort(data, key=str.lower),
["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b", "Z"])
self.assertEqual(algo.NiceSort(data, key=str.upper),
["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b", "Z"])
def testLargeA(self):
data = [
"Eegah9ei", "xij88brTulHYAv8IEOyU", "3jTwJPtrXOY22bwL2YoW",
"Z8Ljf1Pf5eBfNg171wJR", "WvNJd91OoXvLzdEiEXa6", "uHXAyYYftCSG1o7qcCqe",
"xpIUJeVT1Rp", "KOt7vn1dWXi", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH",
"cPRi0lM7HLnSuWA2G9", "KVQqLPDjcPjf8T3oyzjcOsfkb",
"guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx",
]
self.assertEqual(algo.NiceSort(data), [
"3jTwJPtrXOY22bwL2YoW", "Eegah9ei", "KOt7vn1dWXi",
"KVQqLPDjcPjf8T3oyzjcOsfkb", "WvNJd91OoXvLzdEiEXa6",
"Z8Ljf1Pf5eBfNg171wJR", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH",
"cPRi0lM7HLnSuWA2G9", "guKJkXnkULealVC8CyF1xefym",
"pqF8dkU5B1cMnyZuREaSOADYx", "uHXAyYYftCSG1o7qcCqe",
"xij88brTulHYAv8IEOyU", "xpIUJeVT1Rp"
])
def testLargeB(self):
data = [
"inst-0.0.0.0-0.0.0.0",
"inst-0.1.0.0-0.0.0.0",
"inst-0.2.0.0-0.0.0.0",
"inst-0.2.1.0-0.0.0.0",
"inst-0.2.2.0-0.0.0.0",
"inst-0.2.2.0-0.0.0.9",
"inst-0.2.2.0-0.0.3.9",
"inst-0.2.2.0-0.2.0.9",
"inst-0.2.2.0-0.9.0.9",
"inst-0.20.2.0-0.0.0.0",
"inst-0.20.2.0-0.9.0.9",
"inst-10.020.2.0-0.9.0.10",
"inst-15.020.2.0-0.9.1.00",
"inst-100.020.2.0-0.9.0.9",
# Only the last group, not converted to a number anymore, differs
"inst-100.020.2.0a999",
"inst-100.020.2.0b000",
"inst-100.020.2.0c10",
"inst-100.020.2.0c101",
"inst-100.020.2.0c2",
"inst-100.020.2.0c20",
"inst-100.020.2.0c3",
"inst-100.020.2.0c39123",
]
rnd = random.Random(16205)
for _ in range(10):
testdata = data[:]
rnd.shuffle(testdata)
assert testdata != data
self.assertEqual(algo.NiceSort(testdata), data)
class _CallCount:
def __init__(self, fn):
self.count = 0
self.fn = fn
def __call__(self, *args):
self.count += 1
return self.fn(*args)
def testKeyfuncA(self):
# Generate some random numbers
rnd = random.Random(21131)
numbers = [rnd.randint(0, 10000) for _ in range(999)]
assert numbers != sorted(numbers)
# Convert to hex
data = [hex(i) for i in numbers]
datacopy = data[:]
keyfn = self._CallCount(lambda value: str(int(value, 16)))
# Sort with key function converting hex to decimal
result = algo.NiceSort(data, key=keyfn)
self.assertEqual([hex(i) for i in sorted(numbers)], result)
self.assertEqual(data, datacopy, msg="Input data was modified in NiceSort")
self.assertEqual(keyfn.count, len(numbers),
msg="Key function was not called once per value")
class _TestData:
def __init__(self, name, value):
self.name = name
self.value = value
def testKeyfuncB(self):
rnd = random.Random(27396)
data = []
for i in range(123):
v1 = rnd.randint(0, 5)
v2 = rnd.randint(0, 5)
data.append(self._TestData("inst-%s-%s-%s" % (v1, v2, i),
(v1, v2, i)))
rnd.shuffle(data)
assert data != sorted(data, key=operator.attrgetter("name"))
keyfn = self._CallCount(operator.attrgetter("name"))
# Sort by name
result = algo.NiceSort(data, key=keyfn)
self.assertEqual(result, sorted(data, key=operator.attrgetter("value")))
self.assertEqual(keyfn.count, len(data),
msg="Key function was not called once per value")
if __name__ == "__main__":
testutils.GanetiTestProgram()
......@@ -1317,60 +1317,6 @@ class TestNewUUID(unittest.TestCase):
self.failUnless(utils.UUID_RE.match(utils.NewUUID()))
class TestUniqueSequence(unittest.TestCase):
"""Test case for UniqueSequence"""
def _test(self, input, expected):
self.assertEqual(utils.UniqueSequence(input), expected)
def runTest(self):
# Ordered input
self._test([1, 2, 3], [1, 2, 3])
self._test([1, 1, 2, 2, 3, 3], [1, 2, 3])
self._test([1, 2, 2, 3], [1, 2, 3])
self._test([1, 2, 3, 3], [1, 2, 3])
# Unordered input
self._test([1, 2, 3, 1, 2, 3], [1, 2, 3])
self._test([1, 1, 2, 3, 3, 1, 2], [1, 2, 3])
# Strings
self._test(["a", "a"], ["a"])
self._test(["a", "b"], ["a", "b"])
self._test(["a", "b", "a"], ["a", "b"])
class TestFindDuplicates(unittest.TestCase):
"""Test case for FindDuplicates"""
def _Test(self, seq, expected):
result = utils.FindDuplicates(seq)
self.assertEqual(result, utils.UniqueSequence(result))
self.assertEqual(set(result), set(expected))
def test(self):
self._Test([], [])
self._Test([1, 2, 3], [])
self._Test([9, 8, 8, 0, 5, 1, 7, 0, 6, 7], [8, 0, 7])
for exp in [[1, 2, 3], [3, 2, 1]]:
self._Test([1, 1, 2, 2, 3, 3], exp)
self._Test(["A", "a", "B"], [])
self._Test(["a", "A", "a", "B"], ["a"])
self._Test("Hello World out there!", ["e", " ", "o", "r", "t", "l"])
self._Test(self._Gen(False), [])
self._Test(self._Gen(True), range(1, 10))
@staticmethod
def _Gen(dup):
for i in range(10):
yield i
if dup:
for _ in range(i):
yield i
class TestFirstFree(unittest.TestCase):
"""Test case for the FirstFree function"""
......@@ -2663,139 +2609,5 @@ class TestNormalizeAndValidateMac(unittest.TestCase):
self.assertEqual(utils.NormalizeAndValidateMac(mac), mac.lower())
class TestNiceSort(unittest.TestCase):
def test(self):
self.assertEqual(utils.NiceSort([]), [])
self.assertEqual(utils.NiceSort(["foo"]), ["foo"])
self.assertEqual(utils.NiceSort(["bar", ""]), ["", "bar"])
self.assertEqual(utils.NiceSort([",", "."]), [",", "."])
self.assertEqual(utils.NiceSort(["0.1", "0.2"]), ["0.1", "0.2"])
self.assertEqual(utils.NiceSort(["0;099", "0,099", "0.1", "0.2"]),
["0,099", "0.1", "0.2", "0;099"])
data = ["a0", "a1", "a99", "a20", "a2", "b10", "b70", "b00", "0000"]
self.assertEqual(utils.NiceSort(data),
["0000", "a0", "a1", "a2", "a20", "a99",
"b00", "b10", "b70"])
data = ["a0-0", "a1-0", "a99-10", "a20-3", "a0-4", "a99-3", "a09-2",
"Z", "a9-1", "A", "b"]
self.assertEqual(utils.NiceSort(data),
["A", "Z", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b"])
self.assertEqual(utils.NiceSort(data, key=str.lower),
["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b", "Z"])
self.assertEqual(utils.NiceSort(data, key=str.upper),
["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2",
"a20-3", "a99-3", "a99-10", "b", "Z"])
def testLargeA(self):
data = [
"Eegah9ei", "xij88brTulHYAv8IEOyU", "3jTwJPtrXOY22bwL2YoW",
"Z8Ljf1Pf5eBfNg171wJR", "WvNJd91OoXvLzdEiEXa6", "uHXAyYYftCSG1o7qcCqe",
"xpIUJeVT1Rp", "KOt7vn1dWXi", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH",
"cPRi0lM7HLnSuWA2G9", "KVQqLPDjcPjf8T3oyzjcOsfkb",
"guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx",
]
self.assertEqual(utils.NiceSort(data), [
"3jTwJPtrXOY22bwL2YoW", "Eegah9ei", "KOt7vn1dWXi",
"KVQqLPDjcPjf8T3oyzjcOsfkb", "WvNJd91OoXvLzdEiEXa6",
"Z8Ljf1Pf5eBfNg171wJR", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH",
"cPRi0lM7HLnSuWA2G9", "guKJkXnkULealVC8CyF1xefym",
"pqF8dkU5B1cMnyZuREaSOADYx", "uHXAyYYftCSG1o7qcCqe",
"xij88brTulHYAv8IEOyU", "xpIUJeVT1Rp"
])
def testLargeB(self):
data = [
"inst-0.0.0.0-0.0.0.0",
"inst-0.1.0.0-0.0.0.0",
"inst-0.2.0.0-0.0.0.0",
"inst-0.2.1.0-0.0.0.0",
"inst-0.2.2.0-0.0.0.0",
"inst-0.2.2.0-0.0.0.9",
"inst-0.2.2.0-0.0.3.9",
"inst-0.2.2.0-0.2.0.9",
"inst-0.2.2.0-0.9.0.9",
"inst-0.20.2.0-0.0.0.0",
"inst-0.20.2.0-0.9.0.9",
"inst-10.020.2.0-0.9.0.10",
"inst-15.020.2.0-0.9.1.00",
"inst-100.020.2.0-0.9.0.9",
# Only the last group, not converted to a number anymore, differs
"inst-100.020.2.0a999",
"inst-100.020.2.0b000",
"inst-100.020.2.0c10",
"inst-100.020.2.0c101",
"inst-100.020.2.0c2",
"inst-100.020.2.0c20",
"inst-100.020.2.0c3",
"inst-100.020.2.0c39123",
]
rnd = random.Random(16205)
for _ in range(10):
testdata = data[:]
rnd.shuffle(testdata)
assert testdata != data
self.assertEqual(utils.NiceSort(testdata), data)
class _CallCount:
def __init__(self, fn):
self.count = 0
self.fn = fn
def __call__(self, *args):
self.count += 1
return self.fn(*args)
def testKeyfuncA(self):
# Generate some random numbers
rnd = random.Random(21131)