Commit 4e529d66 authored by Ilias Tsitsimpis's avatar Ilias Tsitsimpis

snf-pithos-app: Use the new AstakosClient

parent 625bd291
......@@ -1050,9 +1050,6 @@ this options:
PITHOS_SERVICE_TOKEN = 'pithos_service_token22w'
# Set to False if astakos & pithos are on the same host
PITHOS_PROXY_USER_SERVICES = True
The ``PITHOS_BACKEND_DB_CONNECTION`` option tells to the Pithos app where to
find the Pithos backend database. Above we tell Pithos that its database is
......
......@@ -63,9 +63,6 @@ In `/etc/synnefo/pithos.conf` add:
PITHOS_UPDATE_MD5 = False
PITHOS_SERVICE_TOKEN = 'XXXXXXXXXXX'
# Set False if astakos & pithos are on the same node
PITHOS_PROXY_USER_SERVICES = True
Install pithos web UI with:
......
......@@ -56,7 +56,6 @@ else:
ASTAKOS_IM_MODULES = ['local', 'shibboleth']
PITHOS_PROXY_USER_SERVICES = False
ASTAKOS_BASE_URL = 'http://accounts.example.synnefo.org/astakos/'
ASTAKOS_AUTH_URL = 'http://accounts.example.synnefo.org/astakos/identity/v2.0/'
......
......@@ -2,7 +2,7 @@
#PITHOS_BASE_URL = "https://pithos.example.synnefo.org/"
# Top-level URL for the Astakos instance to be used for user management
#ASTAKOS_BASE_URL = 'https://accounts.example.synnefo.org/'
#ASTAKOS_AUTH_URL = 'https://accounts.example.synnefo.org/identity/v2.0/'
# Cookie name to search for fallback token
#PITHOS_ASTAKOS_COOKIE_NAME = '_pithos2_a'
......@@ -37,12 +37,8 @@
# identity management. WARNING: Setting to True will break your installation.
# PITHOS_TRANSLATE_UUIDS = False
# Set PITHOS_PROXY_USER_SERVICES to True to have snf-pithos-app handle all
# Astakos user-visible services (feedback, login, etc.) by proxying them to
# a running Astakos.
# Set to False if snf astakos-app is running on the same machine, so it handles
# the requests on its own.
#PITHOS_PROXY_USER_SERVICES = True
## Proxy Astakos services under the following path
#PITHOS_PROXY_PREFIX = '_astakos'
# Tune the size of the http pool for the astakos client.
# It limits the maximum number of quota changing requests
......
......@@ -60,7 +60,7 @@ from pithos.api.util import (
)
from pithos.api.settings import (UPDATE_MD5, TRANSLATE_UUIDS,
SERVICE_TOKEN, ASTAKOS_BASE_URL)
SERVICE_TOKEN, ASTAKOS_AUTH_URL)
from pithos.api import settings
......@@ -76,9 +76,9 @@ logger = logging.getLogger(__name__)
def get_uuids(names):
try:
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2,
use_pool=True, logger=logger)
uuids = astakos.service_get_uuids(SERVICE_TOKEN, names)
astakos = AstakosClient(SERVICE_TOKEN, ASTAKOS_AUTH_URL,
retry=2, use_pool=True, logger=logger)
uuids = astakos.service_get_uuids(names)
except Exception, e:
logger.exception(e)
return {}
......@@ -710,7 +710,7 @@ def object_list(request, v_account, v_container):
if name != x[:name_idx]:
continue
objects_bulk.append(x[name_idx:])
if len(objects_bulk) > 0:
object_permissions = \
request.backend.get_object_permissions_bulk(
......
......@@ -58,8 +58,7 @@ class Command(NoArgsCommand):
b = get_backend()
try:
b.pre_exec()
pending_commissions = b.astakosclient.get_pending_commissions(
token=b.service_token)
pending_commissions = b.astakosclient.get_pending_commissions()
if pending_commissions:
self.stdout.write(
......@@ -73,7 +72,6 @@ class Command(NoArgsCommand):
to_accept = b.commission_serials.lookup(pending_commissions)
to_reject = list(set(pending_commissions) - set(to_accept))
response = b.astakosclient.resolve_commissions(
token=b.service_token,
accept_serials=to_accept,
reject_serials=to_reject
)
......
#coding=utf8
from django.conf import settings
from synnefo.lib import parse_base_url, join_urls
from synnefo.lib.services import fill_endpoints, get_public_endpoint
from synnefo.util.keypath import get_path, set_path
from synnefo.lib.services import fill_endpoints
from synnefo.util.keypath import get_path
from pithos.api.services import pithos_services as vanilla_pithos_services
from astakosclient import astakos_services as vanilla_astakos_services
from astakosclient import AstakosClient
from copy import deepcopy
# --------------------------------------------------------------------
# Process Pithos settings
# Top-level URL for Pithos. Must set.
BASE_URL = getattr(settings, 'PITHOS_BASE_URL',
"https://object-store.example.synnefo.org/pithos/")
# Service Token acquired by identity provider.
SERVICE_TOKEN = getattr(settings, 'PITHOS_SERVICE_TOKEN', '')
BASE_HOST, BASE_PATH = parse_base_url(BASE_URL)
# Process Astakos settings
ASTAKOS_BASE_URL = getattr(settings, 'ASTAKOS_BASE_URL',
'https://accounts.example.synnefo.org/astakos/')
ASTAKOS_BASE_HOST, ASTAKOS_BASE_PATH = parse_base_url(ASTAKOS_BASE_URL)
pithos_services = deepcopy(vanilla_pithos_services)
fill_endpoints(pithos_services, BASE_URL)
PITHOS_PREFIX = get_path(pithos_services, 'pithos_object-store.prefix')
PUBLIC_PREFIX = get_path(pithos_services, 'pithos_public.prefix')
UI_PREFIX = get_path(pithos_services, 'pithos_ui.prefix')
astakos_services = deepcopy(vanilla_astakos_services)
fill_endpoints(astakos_services, ASTAKOS_BASE_URL)
CUSTOMIZE_ASTAKOS_SERVICES = getattr(settings,
'PITHOS_CUSTOMIZE_ASTAKOS_SERVICES', ())
for path, value in CUSTOMIZE_ASTAKOS_SERVICES:
set_path(astakos_services, path, value, createpath=True)
COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a')
ASTAKOS_ACCOUNTS_PREFIX = get_path(astakos_services, 'astakos_account.prefix')
ASTAKOS_VIEWS_PREFIX = get_path(astakos_services, 'astakos_ui.prefix')
ASTAKOS_KEYSTONE_PREFIX = get_path(astakos_services, 'astakos_identity.prefix')
# --------------------------------------------------------------------
# Process Astakos settings
BASE_ASTAKOS_PROXY_PATH = getattr(settings, 'PITHOS_BASE_ASTAKOS_PROXY_PATH',
ASTAKOS_BASE_PATH)
BASE_ASTAKOS_PROXY_PATH = join_urls(BASE_PATH, BASE_ASTAKOS_PROXY_PATH)
BASE_ASTAKOS_PROXY_PATH = BASE_ASTAKOS_PROXY_PATH.strip('/')
ASTAKOS_AUTH_URL = getattr(
settings, 'ASTAKOS_AUTH_URL',
'https://accounts.example.synnefo.org/astakos/identity/v2.0/')
ASTAKOSCLIENT_POOLSIZE = \
getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE', 200)
# --------------------------------------
# Define a LazyAstakosUrl
class LazyAstakosUrl(object):
def __init__(self, endpoints_name):
self.endpoints_name = endpoints_name
def __str__(self):
if not hasattr(self, 'str'):
try:
astakos_client = \
AstakosClient(SERVICE_TOKEN, ASTAKOS_AUTH_URL)
self.str = getattr(astakos_client, self.endpoints_name)
except:
return None
return self.str
# --------------------------------------
# Define ASTAKOS_ACCOUNT_URL and ASTAKOS_UR_URL as LazyAstakosUrl
ASTAKOS_ACCOUNT_URL = LazyAstakosUrl('account_url')
ASTAKOS_UI_URL = LazyAstakosUrl('ui_url')
# --------------------------------------
# Define Astakos prefixes
ASTAKOS_PROXY_PREFIX = getattr(settings, 'PITHOS_PROXY_PREFIX', '_astakos')
ASTAKOS_AUTH_PREFIX = join_urls('/', ASTAKOS_PROXY_PREFIX, 'identity')
ASTAKOS_ACCOUNT_PREFIX = join_urls('/', ASTAKOS_PROXY_PREFIX, 'account')
ASTAKOS_UI_PREFIX = join_urls('/', ASTAKOS_PROXY_PREFIX, 'ui')
# --------------------------------------
# Define Astakos proxy paths
ASTAKOS_AUTH_PROXY_PATH = join_urls(BASE_PATH, ASTAKOS_AUTH_PREFIX)
ASTAKOS_ACCOUNT_PROXY_PATH = join_urls(BASE_PATH, ASTAKOS_ACCOUNT_PREFIX)
ASTAKOS_UI_PROXY_PATH = join_urls(BASE_PATH, ASTAKOS_UI_PREFIX)
# Astakos login URL to redirect if the user information is missing
LOGIN_URL = join_urls(ASTAKOS_UI_PROXY_PATH, 'login')
ASTAKOSCLIENT_POOLSIZE = getattr(settings, 'PITHOS_ASTAKOSCLIENT_POOLSIZE',
200)
COOKIE_NAME = getattr(settings, 'PITHOS_ASTAKOS_COOKIE_NAME', '_pithos2_a')
# --------------------------------------------------------------------
# Backend settings
# SQLAlchemy (choose SQLite/MySQL/PostgreSQL).
BACKEND_DB_MODULE = getattr(
......@@ -88,9 +122,6 @@ BACKEND_POOL_SIZE = getattr(settings, 'PITHOS_BACKEND_POOL_SIZE', 5)
# Update object checksums.
UPDATE_MD5 = getattr(settings, 'PITHOS_UPDATE_MD5', False)
# Service Token acquired by identity provider.
SERVICE_TOKEN = getattr(settings, 'PITHOS_SERVICE_TOKEN', '')
RADOS_STORAGE = getattr(settings, 'PITHOS_RADOS_STORAGE', False)
RADOS_POOL_BLOCKS = getattr(settings, 'PITHOS_RADOS_POOL_BLOCKS', 'blocks')
RADOS_POOL_MAPS = getattr(settings, 'PITHOS_RADOS_POOL_MAPS', 'maps')
......@@ -99,13 +130,6 @@ RADOS_POOL_MAPS = getattr(settings, 'PITHOS_RADOS_POOL_MAPS', 'maps')
# identity management. WARNING: Setting to True will break your installation.
TRANSLATE_UUIDS = getattr(settings, 'PITHOS_TRANSLATE_UUIDS', False)
# Set PROXY_USER_SERVICES to True to have snf-pithos-app handle all Astakos
# user-visible services (feedback, login, etc.) by proxying them to a running
# Astakos.
# Set to False if snf astakos-app is running on the same machine, so it handles
# the requests on its own.
PROXY_USER_SERVICES = getattr(settings, 'PITHOS_PROXY_USER_SERVICES', True)
# Set how many random bytes to use for constructing the URL
# of Pithos public files
PUBLIC_URL_SECURITY = getattr(settings, 'PITHOS_PUBLIC_URL_SECURITY', 16)
......@@ -125,8 +149,3 @@ BACKEND_BLOCK_SIZE = getattr(
# The backend block hash algorithm
BACKEND_HASH_ALGORITHM = getattr(
settings, 'PITHOS_BACKEND_HASH_ALGORITHM', 'sha256')
ASTAKOS_UI_URL = get_public_endpoint(astakos_services, 'astakos_ui', '').rstrip('/')
# Astakos login URL to redirect if the user information is missing
LOGIN_URL = join_urls(ASTAKOS_UI_URL, 'login')
......@@ -40,10 +40,11 @@ from snf_django.lib.api.urls import api_patterns
from snf_django.lib.api import api_endpoint_not_found
from snf_django.utils.urls import extend_endpoint_with_slash
from pithos.api.settings import (
pithos_services,
BASE_PATH, ASTAKOS_BASE_URL, BASE_ASTAKOS_PROXY_PATH,
ASTAKOS_ACCOUNTS_PREFIX, PROXY_USER_SERVICES,
PITHOS_PREFIX, PUBLIC_PREFIX, UI_PREFIX)
BASE_PATH, PITHOS_PREFIX, PUBLIC_PREFIX, UI_PREFIX,
ASTAKOS_AUTH_PROXY_PATH, ASTAKOS_AUTH_URL,
ASTAKOS_ACCOUNT_PROXY_PATH, ASTAKOS_ACCOUNT_URL,
ASTAKOS_UI_PROXY_PATH, ASTAKOS_UI_URL,
pithos_services)
urlpatterns = []
......@@ -83,19 +84,25 @@ urlpatterns += patterns(
(prefix_pattern(BASE_PATH), include(pithos_patterns)),
)
if PROXY_USER_SERVICES:
astakos_proxy = partial(proxy, proxy_base=BASE_ASTAKOS_PROXY_PATH,
target_base=ASTAKOS_BASE_URL)
proxy_patterns = api_patterns(
'',
(r'^login/?$', astakos_proxy),
(r'^feedback/?$', astakos_proxy),
(r'^user_catalogs/?$', astakos_proxy),
(prefix_pattern(ASTAKOS_ACCOUNTS_PREFIX), astakos_proxy),
)
# --------------------------------------
# PROXY settings
astakos_auth_proxy = \
partial(proxy, proxy_base=ASTAKOS_AUTH_PROXY_PATH,
target_base=ASTAKOS_AUTH_URL)
astakos_account_proxy = \
partial(proxy, proxy_base=ASTAKOS_ACCOUNT_PROXY_PATH,
target_base=ASTAKOS_ACCOUNT_URL)
astakos_ui_proxy = \
partial(proxy, proxy_base=ASTAKOS_UI_PROXY_PATH,
target_base=ASTAKOS_UI_URL)
urlpatterns += patterns(
'',
(prefix_pattern(BASE_ASTAKOS_PROXY_PATH), include(proxy_patterns)),
)
urlpatterns += api_patterns(
'',
(prefix_pattern(ASTAKOS_AUTH_PROXY_PATH), astakos_auth_proxy),
(prefix_pattern(ASTAKOS_ACCOUNT_PROXY_PATH), astakos_account_proxy),
)
urlpatterns += patterns(
'',
(prefix_pattern(ASTAKOS_UI_PROXY_PATH), astakos_ui_proxy),
)
......@@ -56,7 +56,7 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
BACKEND_QUEUE_EXCHANGE,
ASTAKOSCLIENT_POOLSIZE,
SERVICE_TOKEN,
ASTAKOS_BASE_URL,
ASTAKOS_AUTH_URL,
BACKEND_ACCOUNT_QUOTA,
BACKEND_CONTAINER_QUOTA,
BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
......@@ -111,7 +111,7 @@ def printable_header_dict(d):
for timestamp in timestamps:
if timestamp in d and d[timestamp]:
d[timestamp] = utils.isoformat(
datetime.fromtimestamp(d[timestamp]))
datetime.fromtimestamp(d[timestamp]))
return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
......@@ -298,10 +298,11 @@ def is_uuid(str):
##########################
def retrieve_displayname(token, uuid, fail_silently=True):
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
retry=2, use_pool=True,
logger=logger)
try:
displayname = astakos.get_username(token, uuid)
displayname = astakos.get_username(uuid)
except NoUserName:
if not fail_silently:
raise ItemNotExists(uuid)
......@@ -312,9 +313,10 @@ def retrieve_displayname(token, uuid, fail_silently=True):
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
retry=2, use_pool=True,
logger=logger)
catalog = astakos.get_usernames(token, uuids) or {}
catalog = astakos.get_usernames(uuids) or {}
missing = list(set(uuids) - set(catalog))
if missing and not fail_silently:
raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
......@@ -325,19 +327,21 @@ def retrieve_uuid(token, displayname):
if is_uuid(displayname):
return displayname
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
retry=2, use_pool=True,
logger=logger)
try:
uuid = astakos.get_uuid(token, displayname)
uuid = astakos.get_uuid(displayname)
except NoUUID:
raise ItemNotExists(displayname)
return uuid
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
retry=2, use_pool=True,
logger=logger)
catalog = astakos.get_uuids(token, displaynames) or {}
catalog = astakos.get_uuids(displaynames) or {}
missing = list(set(displaynames) - set(catalog))
if missing and not fail_silently:
raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
......@@ -1010,7 +1014,7 @@ BACKEND_KWARGS = dict(
queue_module=BACKEND_QUEUE_MODULE,
queue_hosts=BACKEND_QUEUE_HOSTS,
queue_exchange=BACKEND_QUEUE_EXCHANGE,
astakos_url=ASTAKOS_BASE_URL,
astakos_auth_url=ASTAKOS_AUTH_URL,
service_token=SERVICE_TOKEN,
astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
free_versioning=BACKEND_FREE_VERSIONING,
......@@ -1063,9 +1067,10 @@ def update_response_headers(request, response):
def get_pithos_usage(token):
"""Get Pithos Usage from astakos."""
astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
retry=2, use_pool=True,
logger=logger)
quotas = astakos.get_quotas(token)['system']
quotas = astakos.get_quotas()['system']
pithos_resources = [r['name'] for r in resources]
map(quotas.pop, filter(lambda k: k not in pithos_resources, quotas.keys()))
return quotas.popitem()[-1] # assume only one resource
......@@ -1080,7 +1085,7 @@ def api_method(http_method=None, token_required=True, user_required=True,
@api.api_method(http_method=http_method, token_required=token_required,
user_required=user_required,
logger=logger, format_allowed=format_allowed,
astakos_url=ASTAKOS_BASE_URL,
astakos_auth_url=ASTAKOS_AUTH_URL,
serializations=serializations,
strict_serlization=strict_serlization)
@wraps(func)
......
......@@ -36,13 +36,12 @@ import sqlalchemy as sa
catalog = {}
def _get_uuid(account, service_token, astakos_client):
def _get_uuid(account, astakos_client):
global catalog
if account in catalog:
return catalog[account]
try:
catalog[account] = astakos_client.service_get_uuid(service_token,
account)
catalog[account] = astakos_client.service_get_uuid(account)
print '\n', account, '-->', catalog[account]
except NoUUID:
return None
......@@ -54,13 +53,12 @@ def _get_uuid(account, service_token, astakos_client):
inverse_catalog = {}
def _get_displayname(account, service_token, astakos_client):
def _get_displayname(account, astakos_client):
global inverse_catalog
if account in inverse_catalog:
return inverse_catalog[account]
try:
inverse_catalog[account] = astakos_client.service_get_username(
service_token, account)
inverse_catalog[account] = astakos_client.service_get_username(account)
print '\n', account, '-->', inverse_catalog[account]
except NoUserName:
return None
......@@ -221,9 +219,9 @@ def upgrade():
except ImportError:
return
else:
astakos_client = AstakosClient(settings.ASTAKOS_BASE_URL,
retry=3,
use_pool=True)
astakos_client = AstakosClient(
settings.SERVICE_TOKEN, settings.ASTAKOS_AUTH_URL,
retry=3, use_pool=True)
get_uuid = functools.partial(_get_uuid,
service_token=settings.SERVICE_TOKEN,
astakos_client=astakos_client)
......@@ -236,11 +234,10 @@ def downgrade():
except ImportError:
return
else:
astakos_client = AstakosClient(settings.ASTAKOS_BASE_URL,
retry=3,
use_pool=True)
astakos_client = AstakosClient(
settings.SERVICE_TOKEN, settings.ASTAKOS_AUTH_URL,
retry=3, use_pool=True)
get_displayname = functools.partial(
_get_displayname,
service_token=settings.SERVICE_TOKEN,
astakos_client=astakos_client)
migrate(get_displayname)
......@@ -151,7 +151,7 @@ class ModularBackend(BaseBackend):
block_module=None, block_path=None, block_umask=None,
block_size=None, hash_algorithm=None,
queue_module=None, queue_hosts=None, queue_exchange=None,
astakos_url=None, service_token=None,
astakos_auth_url=None, service_token=None,
astakosclient_poolsize=None,
free_versioning=True, block_params=None,
public_url_security=None,
......@@ -236,17 +236,17 @@ class ModularBackend(BaseBackend):
self.queue = NoQueue()
self.astakos_url = astakos_url
self.astakos_auth_url = astakos_auth_url
self.service_token = service_token
if not astakos_url or not AstakosClient:
if not astakos_auth_url or not AstakosClient:
self.astakosclient = DisabledAstakosClient(
astakos_url,
service_token, astakos_auth_url,
use_pool=True,
pool_size=astakosclient_poolsize)
else:
self.astakosclient = AstakosClient(
astakos_url,
service_token, astakos_auth_url,
use_pool=True,
pool_size=astakosclient_poolsize)
......@@ -280,7 +280,6 @@ class ModularBackend(BaseBackend):
self.wrapper.execute()
r = self.astakosclient.resolve_commissions(
token=self.service_token,
accept_serials=self.serials,
reject_serials=[])
self.commission_serials.delete_many(
......@@ -290,7 +289,6 @@ class ModularBackend(BaseBackend):
else:
if self.serials:
self.astakosclient.resolve_commissions(
token=self.service_token,
accept_serials=[],
reject_serials=self.serials)
self.commission_serials.delete_many(
......@@ -1577,7 +1575,6 @@ class ModularBackend(BaseBackend):
try:
name = details['path'] if 'path' in details else ''
serial = self.astakosclient.issue_one_commission(
token=self.service_token,
holder=account,
source=DEFAULT_SOURCE,
provisions={'pithos.diskspace': size},
......
......@@ -46,7 +46,7 @@ class PithosBackendPool(ObjectPool):
block_size=None, hash_algorithm=None,
queue_module=None, queue_hosts=None,
queue_exchange=None, free_versioning=True,
astakos_url=None, service_token=None,
astakos_auth_url=None, service_token=None,
astakosclient_poolsize=None,
block_params=None,
public_url_security=None,
......@@ -66,7 +66,7 @@ class PithosBackendPool(ObjectPool):
self.block_params = block_params
self.queue_hosts = queue_hosts
self.queue_exchange = queue_exchange
self.astakos_url = astakos_url
self.astakos_auth_url = astakos_auth_url
self.service_token = service_token
self.astakosclient_poolsize = astakosclient_poolsize
self.free_versioning = free_versioning
......@@ -89,7 +89,7 @@ class PithosBackendPool(ObjectPool):
block_params=self.block_params,
queue_hosts=self.queue_hosts,
queue_exchange=self.queue_exchange,
astakos_url=self.astakos_url,
astakos_auth_url=self.astakos_auth_url,
service_token=self.service_token,
astakosclient_poolsize=self.astakosclient_poolsize,
free_versioning=self.free_versioning,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment