Commit ee6f0a62 authored by Sofia Papagiannaki's avatar Sofia Papagiannaki
Browse files

Merge branch 'feature-pithos-container-quota' into develop

parents ab73ab61 768c9b0a
......@@ -3,8 +3,8 @@ Changelog
v0.14.0
------
(* Unreleased *)
* settings *PITHOS_BACKEND_POOL_SIZE*
* new settings *PITHOS_BACKEND_POOL_SIZE*, *PITHOS_BACKEND_ACCOUNT_QUOTA*, *PITHOS_BACKEND_CONTAINER_QUOTA*
* setting *PITHOS_BACKEND_QUOTA* is substituted by the setting *PITHOS_BACKEND_ACCOUNT_QUOTA*
* setting *PITHOS_UPDATE_MD5* is set to False by default
* Remove PITHOS_AUTHENTICATION_USERS setting
......
......@@ -64,7 +64,7 @@ In `/etc/synnefo/pithos.conf` add:
PITHOS_BACKEND_DB_CONNECTION = 'postgresql://synnefo:example_passw0rd@db.example.com:5432/snf_pithos'
PITHOS_BACKEND_BLOCK_PATH = '/srv/pithos/data'
PITHOS_BACKEND_QUOTA = 20 * 1024 * 1024 * 1024
PITHOS_BACKEND_ACCOUNT_QUOTA = 20 * 1024 * 1024 * 1024
PITHOS_UPDATE_MD5 = False
PITHOS_SERVICE_TOKEN = 'XXXXXXXXXXX'
......
......@@ -40,7 +40,8 @@ PITHOS_BACKEND_BLOCK_UMASK 0o022
PITHOS_BACKEND_QUEUE_MODULE None Use ``pithos.backends.lib.rabbitmq`` to enable
PITHOS_BACKEND_QUEUE_HOSTS None Format like [``amqp://guest:guest@localhost:5672``
PITHOS_BACKEND_QUEUE_EXCHANGE pithos
PITHOS_BACKEND_QUOTA 50 GB (50 * 1024 ** 3) Default user quota
PITHOS_BACKEND_ACCOUNT_QUOTA 50 GB (50 * 1024 ** 3) Default account quota (set to 0 to disable control)
PITHOS_BACKEND_CONTAINER_QUOTA 0 Default container quota (set to 0 to disable control - default disabled)
PITHOS_BACKEND_VERSIONING auto Default versioning policy for containers
PITHOS_BACKEND_FREE_VERSIONING True Default versioning debit policy (default free)
PITHOS_BACKEND_POOL_SIZE 5 Default backend pool size
......
......@@ -13,7 +13,8 @@
#PITHOS_BACKEND_BLOCK_UMASK = 0o022
# Default setting for new accounts.
#PITHOS_BACKEND_QUOTA = 50 * 1024 * 1024 * 1024
#PITHOS_BACKEND_ACCOUNT_QUOTA = 50 * 1024 * 1024 * 1024
#PITHOS_BACKEND_CONTAINER_QUOTA = 0
#PITHOS_BACKEND_VERSIONING = 'auto'
#PITHOS_BACKEND_FREE_VERSIONING = True
......
from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING)
#from pithos.backends import connect_backend
from pithos.api.util import hashmap_md5, get_backend
from django.core.mail import send_mail
from django.utils.translation import ugettext as _
from astakos.im.settings import DEFAULT_FROM_EMAIL
from django.conf import settings
import socket
from smtplib import SMTPException
......@@ -17,8 +14,6 @@ def update_md5(m):
return
backend = get_backend()
backend.default_policy['quota'] = BACKEND_QUOTA
backend.default_policy['versioning'] = BACKEND_VERSIONING
path = m['value']
account, container, name = path.split('/', 2)
......@@ -50,9 +45,11 @@ def send_sharing_notification(m):
account, container, name = path.split('/', 2)
subject = 'Invitation to a Pithos+ shared object'
from_email = DEFAULT_FROM_EMAIL
from_email = settings.SERVER_EMAIL
recipient_list = members
message = 'User %s has invited you to a Pithos+ shared object. You can view it under "Shared to me" at "%s".' % (user, path)
message = ("User %s has invited you to a Pithos+ shared object."
"You can view it under \"Shared to me\" at \"%s\".")
message = message % (user, path)
try:
send_mail(subject, message, from_email, recipient_list)
print 'INFO: Sharing notification sent for path "%s" to %s' % (
......
......@@ -35,10 +35,9 @@ from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING)
from pithos.api.util import get_backend
class Command(BaseCommand):
args = "<user>"
help = "Get/set a user's quota"
......@@ -63,8 +62,6 @@ class Command(BaseCommand):
raise CommandError("Invalid quota")
backend = get_backend()
backend.default_policy['quota'] = BACKEND_QUOTA
backend.default_policy['versioning'] = BACKEND_VERSIONING
if backend.using_external_quotaholder:
raise CommandError("The system uses an extrenal quota holder.")
......
......@@ -26,8 +26,10 @@ BACKEND_QUEUE_HOSTS = getattr(settings, 'PITHOS_BACKEND_QUEUE_HOSTS', None) # Ex
BACKEND_QUEUE_EXCHANGE = getattr(settings, 'PITHOS_BACKEND_QUEUE_EXCHANGE', 'pithos')
# Default setting for new accounts.
BACKEND_QUOTA = getattr(
settings, 'PITHOS_BACKEND_QUOTA', 50 * 1024 * 1024 * 1024)
BACKEND_ACCOUNT_QUOTA = getattr(
settings, 'PITHOS_BACKEND_ACCOUNT_QUOTA', 50 * 1024 * 1024 * 1024)
BACKEND_CONTAINER_QUOTA = getattr(
settings, 'PITHOS_BACKEND_CONTAINER_QUOTA', 0)
BACKEND_VERSIONING = getattr(settings, 'PITHOS_BACKEND_VERSIONING', 'auto')
BACKEND_FREE_VERSIONING = getattr(settings, 'PITHOS_BACKEND_FREE_VERSIONING', True)
......
......@@ -58,7 +58,8 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
QUOTAHOLDER_URL, QUOTAHOLDER_TOKEN,
QUOTAHOLDER_POOLSIZE,
ASTAKOS_URL,
BACKEND_QUOTA, BACKEND_VERSIONING,
BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
BACKEND_VERSIONING,
BACKEND_FREE_VERSIONING, BACKEND_POOL_SIZE,
COOKIE_NAME, USER_CATALOG_URL,
RADOS_STORAGE, RADOS_POOL_BLOCKS,
......@@ -959,13 +960,14 @@ _pithos_backend_pool = PithosBackendPool(
free_versioning=BACKEND_FREE_VERSIONING,
block_params=BLOCK_PARAMS,
public_url_security=PUBLIC_URL_SECURITY,
public_url_alphabet=PUBLIC_URL_ALPHABET)
public_url_alphabet=PUBLIC_URL_ALPHABET,
account_quota_policy=BACKEND_ACCOUNT_QUOTA,
container_quota_policy=BACKEND_CONTAINER_QUOTA,
container_versioning_policy=BACKEND_VERSIONING)
def get_backend():
backend = _pithos_backend_pool.pool_get()
backend.default_policy['quota'] = BACKEND_QUOTA
backend.default_policy['versioning'] = BACKEND_VERSIONING
backend.messages = []
return backend
......
......@@ -32,8 +32,9 @@
# or implied, of GRNET S.A.
# Default setting for new accounts.
DEFAULT_QUOTA = 0 # No quota.
DEFAULT_VERSIONING = 'auto'
DEFAULT_ACCOUNT_QUOTA = 0 # No quota.
DEFAULT_CONTAINER_QUOTA = 0 # No quota.
DEFAULT_CONTAINER_VERSIONING = 'auto'
......@@ -84,7 +85,8 @@ class BaseBackend(object):
'block_size': Suggested is 4MB
'default_policy': A dictionary with default policy settings
'default_account_policy': A dictionary with default account policy settings
'default_container_policy': A dictionary with default container policy settings
"""
def close(self):
......
......@@ -32,8 +32,6 @@
# or implied, of GRNET S.A.
import sys
import os
import time
import uuid as uuidlib
import logging
import hashlib
......@@ -41,8 +39,10 @@ import binascii
from synnefo.lib.quotaholder import QuotaholderClient
from base import DEFAULT_QUOTA, DEFAULT_VERSIONING, NotAllowedError, QuotaError, BaseBackend, \
AccountExists, ContainerExists, AccountNotEmpty, ContainerNotEmpty, ItemNotExists, VersionNotExists
from base import (DEFAULT_ACCOUNT_QUOTA, DEFAULT_CONTAINER_QUOTA,
DEFAULT_CONTAINER_VERSIONING, NotAllowedError, QuotaError,
BaseBackend, AccountExists, ContainerExists, AccountNotEmpty,
ContainerNotEmpty, ItemNotExists, VersionNotExists)
# Stripped-down version of the HashMap class found in tools.
......@@ -155,7 +155,10 @@ class ModularBackend(BaseBackend):
quotaholder_client_poolsize=None,
free_versioning=True, block_params=None,
public_url_security=None,
public_url_alphabet=None):
public_url_alphabet=None,
account_quota_policy=None,
container_quota_policy=None,
container_versioning_policy=None):
db_module = db_module or DEFAULT_DB_MODULE
db_connection = db_connection or DEFAULT_DB_CONNECTION
block_module = block_module or DEFAULT_BLOCK_MODULE
......@@ -163,8 +166,17 @@ class ModularBackend(BaseBackend):
block_umask = block_umask or DEFAULT_BLOCK_UMASK
block_params = block_params or DEFAULT_BLOCK_PARAMS
#queue_module = queue_module or DEFAULT_QUEUE_MODULE
self.default_policy = {'quota': DEFAULT_QUOTA, 'versioning': DEFAULT_VERSIONING}
account_quota_policy = account_quota_policy or DEFAULT_ACCOUNT_QUOTA
container_quota_policy = container_quota_policy \
or DEFAULT_CONTAINER_QUOTA
container_versioning_policy = container_versioning_policy \
or DEFAULT_CONTAINER_VERSIONING
self.default_account_policy = {'quota': account_quota_policy}
self.default_container_policy = {
'quota': container_quota_policy,
'versioning': container_versioning_policy
}
#queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS
#queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE
......@@ -175,9 +187,6 @@ class ModularBackend(BaseBackend):
self.block_size = 4 * 1024 * 1024 # 4MB
self.free_versioning = free_versioning
self.default_policy = {'quota': DEFAULT_QUOTA,
'versioning': DEFAULT_VERSIONING}
def load_module(m):
__import__(m)
return sys.modules[m]
......@@ -342,7 +351,7 @@ class ModularBackend(BaseBackend):
raise NotAllowedError
return {}
path, node = self._lookup_account(account, True)
policy = self._get_policy(node)
policy = self._get_policy(node, is_account_policy=True)
if self.using_external_quotaholder:
external_quota = external_quota or {}
policy['quota'] = external_quota.get('maxValue', 0)
......@@ -357,8 +366,8 @@ class ModularBackend(BaseBackend):
if user != account:
raise NotAllowedError
path, node = self._lookup_account(account, True)
self._check_policy(policy)
self._put_policy(node, policy, replace)
self._check_policy(policy, is_account_policy=True)
self._put_policy(node, policy, replace, is_account_policy=True)
@backend_method
def put_account(self, user, account, policy=None):
......@@ -372,9 +381,9 @@ class ModularBackend(BaseBackend):
if node is not None:
raise AccountExists('Account already exists')
if policy:
self._check_policy(policy)
self._check_policy(policy, is_account_policy=True)
node = self._put_path(user, self.ROOTNODE, account)
self._put_policy(node, policy, True)
self._put_policy(node, policy, True, is_account_policy=True)
@backend_method
def delete_account(self, user, account):
......@@ -483,7 +492,8 @@ class ModularBackend(BaseBackend):
src_version_id, dest_version_id = self._put_metadata(
user, node, domain, meta, replace)
if src_version_id is not None:
versioning = self._get_policy(node)['versioning']
versioning = self._get_policy(
node, is_account_policy=False)['versioning']
if versioning != 'auto':
self.node.version_remove(src_version_id)
......@@ -498,7 +508,7 @@ class ModularBackend(BaseBackend):
raise NotAllowedError
return {}
path, node = self._lookup_container(account, container)
return self._get_policy(node)
return self._get_policy(node, is_account_policy=False)
@backend_method
def update_container_policy(self, user, account, container, policy, replace=False):
......@@ -509,8 +519,8 @@ class ModularBackend(BaseBackend):
if user != account:
raise NotAllowedError
path, node = self._lookup_container(account, container)
self._check_policy(policy)
self._put_policy(node, policy, replace)
self._check_policy(policy, is_account_policy=False)
self._put_policy(node, policy, replace, is_account_policy=False)
@backend_method
def put_container(self, user, account, container, policy=None):
......@@ -528,11 +538,11 @@ class ModularBackend(BaseBackend):
else:
raise ContainerExists('Container already exists')
if policy:
self._check_policy(policy)
self._check_policy(policy, is_account_policy=False)
path = '/'.join((account, container))
node = self._put_path(
user, self._lookup_account(account, True)[1], path)
self._put_policy(node, policy, True)
self._put_policy(node, policy, True, is_account_policy=False)
@backend_method
def delete_container(self, user, account, container, until=None, prefix='', delimiter=None):
......@@ -865,6 +875,7 @@ class ModularBackend(BaseBackend):
account_path, account_node = self._lookup_account(account, True)
container_path, container_node = self._lookup_container(
account, container)
path, node = self._put_object_node(
container_path, container_node, name)
pre_version_id, dest_version_id = self._put_version_duplicate(user, node, src_node=src_node, size=size, type=type, hash=hash, checksum=checksum, is_copy=is_copy)
......@@ -877,24 +888,35 @@ class ModularBackend(BaseBackend):
del_size = self._apply_versioning(account, container, pre_version_id)
size_delta = size - del_size
if not self.using_external_quotaholder: # Check account quota.
if size_delta > 0:
account_quota = long(self._get_policy(account_node)['quota'])
account_usage = self._get_statistics(account_node)[1] + size_delta
if size_delta > 0:
# Check account quota.
if not self.using_external_quotaholder:
account_quota = long(
self._get_policy(account_node, is_account_policy=True
)['quota']
)
account_usage = self._get_statistics(account_node)[1]
if (account_quota > 0 and account_usage > account_quota):
raise QuotaError('account quota exceeded: limit: %s, usage: %s' % (
account_quota, account_usage
))
# Check container quota.
container_quota = long(self._get_policy(container_node)['quota'])
container_usage = self._get_statistics(container_node)[1] + size_delta
if (container_quota > 0 and container_usage > container_quota):
# This must be executed in a transaction, so the version is
# never created if it fails.
raise QuotaError('container quota exceeded: limit: %s, usage: %s' % (
container_quota, container_usage
))
raise QuotaError(
'Account quota exceeded: limit: %s, usage: %s' % (
account_quota, account_usage
)
)
# Check container quota.
container_quota = long(
self._get_policy(container_node, is_account_policy=False
)['quota']
)
container_usage = self._get_statistics(container_node)[1]
if (container_quota > 0 and container_usage > container_quota):
# This must be executed in a transaction, so the version is
# never created if it fails.
raise QuotaError(
'Container quota exceeded: limit: %s, usage: %s' % (
container_quota, container_usage
)
)
self._report_size_change(user, account, size_delta,
{'action': 'object update', 'path': path,
......@@ -1366,10 +1388,12 @@ class ModularBackend(BaseBackend):
# Policy functions.
def _check_policy(self, policy):
def _check_policy(self, policy, is_account_policy=True):
default_policy = self.default_account_policy \
if is_account_policy else self.default_container_policy
for k in policy.keys():
if policy[k] == '':
policy[k] = self.default_policy.get(k)
policy[k] = default_policy.get(k)
for k, v in policy.iteritems():
if k == 'quota':
q = int(v) # May raise ValueError.
......@@ -1381,15 +1405,19 @@ class ModularBackend(BaseBackend):
else:
raise ValueError
def _put_policy(self, node, policy, replace):
def _put_policy(self, node, policy, replace, is_account_policy=True):
default_policy = self.default_account_policy \
if is_account_policy else self.default_container_policy
if replace:
for k, v in self.default_policy.iteritems():
for k, v in default_policy.iteritems():
if k not in policy:
policy[k] = v
self.node.policy_set(node, policy)
def _get_policy(self, node):
policy = self.default_policy.copy()
def _get_policy(self, node, is_account_policy=True):
default_policy = self.default_account_policy \
if is_account_policy else self.default_container_policy
policy = default_policy.copy()
policy.update(self.node.policy_get(node))
return policy
......@@ -1401,7 +1429,8 @@ class ModularBackend(BaseBackend):
if version_id is None:
return 0
path, node = self._lookup_container(account, container)
versioning = self._get_policy(node)['versioning']
versioning = self._get_policy(
node, is_account_policy=False)['versioning']
if versioning != 'auto':
hash, size = self.node.version_remove(version_id)
self.store.map_delete(hash)
......
......@@ -50,7 +50,11 @@ class PithosBackendPool(ObjectPool):
quotaholder_client_poolsize=None,
block_params=None,
public_url_security=None,
public_url_alphabet=None):
public_url_alphabet=None,
account_quota_policy=None,
container_quota_policy=None,
container_versioning_policy=None
):
super(PithosBackendPool, self).__init__(size=size)
self.db_module = db_module
self.db_connection = db_connection
......@@ -66,8 +70,11 @@ class PithosBackendPool(ObjectPool):
self.quotaholder_token = quotaholder_token
self.quotaholder_client_poolsize = quotaholder_client_poolsize
self.free_versioning = free_versioning
self.public_url_security=public_url_security
self.public_url_alphabet=public_url_alphabet
self.public_url_security = public_url_security
self.public_url_alphabet = public_url_alphabet
self.account_quota_policy = account_quota_policy
self.container_quota_policy = container_quota_policy
self.container_versioning_policy = container_versioning_policy
def _pool_create(self):
backend = connect_backend(
......@@ -86,7 +93,10 @@ class PithosBackendPool(ObjectPool):
quotaholder_client_poolsize=self.quotaholder_client_poolsize,
free_versioning=self.free_versioning,
public_url_security=self.public_url_security,
public_url_alphabet=self.public_url_alphabet)
public_url_alphabet=self.public_url_alphabet,
account_quota_policy=self.account_quota_policy,
container_quota_policy=self.container_quota_policy,
container_versioning_policy=self.container_versioning_policy)
backend._real_close = backend.close
backend.close = instancemethod(_pooled_backend_close, backend,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment