Commit ff5fb93f authored by Filippos Giannakos's avatar Filippos Giannakos

Merge branch 'xseg-refactor'

parents f7a50956 3567078f
......@@ -9,3 +9,8 @@
*.o.cmd
.*.ko.cmd
*.ko
*.pyc
config.mk
*tmp_versions*
*Module*
*modules*
.PHONY: clean
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
#
XSEG_HOME=$(shell pwd)
export XSEG_HOME
.PHONY: default clean target-list distclean
include config.mk
include base.mk
TARGET=default
include ./base.mk
default:
for f in $(MAKEDIRS); do \
make -C $$f || break; \
echo "-------------------------------------------------------------------"; \
echo " "; \
done
FILES="base.mk"
FILES+="config.env"
FILES+="envsetup"
FILES+="Makefile"
FILES+="launch"
clean:
for f in $(MAKEDIRS); do \
make -C $$f clean || break; \
echo " "; \
done
XSEG_BUILD_TARGETS=$(patsubst %,build-%,$(XSEG_DIRS))
XSEG_CLEAN_TARGETS=$(patsubst %,clean-%,$(XSEG_DIRS))
XSEG_INSTALL_TARGETS=$(patsubst %,install-%,$(XSEG_DIRS))
XSEG_INSTALLSRC_TARGETS=$(patsubst %,install-src-%,$(XSEG_DIRS))
default: targets
target-list:
@echo $(XSEG_DOMAIN_TARGETS)
targets: $(XSEG_BUILD_TARGETS)
clean: $(XSEG_CLEAN_TARGETS)
rm $(CONFIG)
install: $(XSEG_INSTALL_TARGETS)
install-src: $(XSEG_INSTALLSRC_TARGETS) $(XSEG_INSTALLSRC_DIRS)
install -d $(DESTDIR)$(srcdir)/lib
install -d $(DESTDIR)$(srcdir)/lib/user
install -d $(DESTDIR)$(srcdir)/lib/kernel
install -d $(DESTDIR)$(srcdir)/
install -o 0 -g 0 -m 755 -t $(DESTDIR)$(srcdir) envsetup
install -o 0 -g 0 -m 755 -t $(DESTDIR)$(srcdir) launch
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir) config.env
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir) Makefile
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir) base.mk
# @for f in $(FILES); do \
# install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir) $$f ; \
# done
.PHONY: $(XSEG_BUILD_TARGETS)
$(XSEG_BUILD_TARGETS):
@echo "" ;
@echo "=== $@ ===";
@make -C $(@:build-%=%) default || exit 1;
@echo "" ;
.PHONY: $(XSEG_INSTALL_TARGETS)
$(XSEG_INSTALL_TARGETS):
@echo "" ;
@echo "=== $@ ===";
@make -C $(@:install-%=%) install || true;
@echo "" ;
.PHONY: $(XSEG_INSTALLSRC_TARGETS)
$(XSEG_INSTALLSRC_TARGETS):
@echo "" ;
@echo "=== $@ ===";
@make -C $(@:install-src-%=%) install-src || true;
@echo "" ;
.PHONY: $(XSEG_CLEAN_TARGETS)
$(XSEG_CLEAN_TARGETS):
@echo "" ;
@echo "=== $@ ===";
@make -C $(@:clean-%=%) clean || true;
@echo "" ;
distclean: $(XSEG_CLEAN_TARGETS)
rm -f lib/user/*.so* lib/user/*.a
rm -f lib/kernel/*.ko
rm -f $(XSEG_HOME)/config.mk
# Default setup for subdirectory Makefiles.
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
#
CC=gcc
# Setup for xseg Makefiles.
ifndef TARGET
TARGET:=$(shell basename $(shell pwd))
endif
export CC=gcc
ifndef MOPTS
MOPTS=
export MOPTS=
endif
ifndef COPTS
COPTS=-O2 -g -finline-functions $(MOPTS) $(DEBUG)
export COPTS=-O2 -g -finline-functions $(MOPTS) $(DEBUG)
endif
ifndef CSTD
CSTD=-std=gnu99 -pedantic
export CSTD=-std=gnu99 -pedantic
endif
bindir=/usr/bin/
libdir=/usr/lib/
pythondir=/usr/lib/python2.7/
moduledir=/lib/modules/`uname -r`/extra/
srcdir=/usr/src/archipelago-kernel-dkms-VERSION/
ganetidir=/usr/share/ganeti/extstorage/vlmc/
INC=-I$(BASE)
CFLAGS=-Wall $(INC) $(COPTS) $(CSTD)
INC+=-I$(BASE)/peers/$(TARGET)
INC+=-I$(BASE)/sys/$(TARGET)
INC+=-I$(BASE)/drivers/$(TARGET)
export INC
export LIB=$(BASE)/lib/$(TARGET)
export CFLAGS=-Wall $(COPTS) $(CSTD)
#ifeq (,$(XSEG_HOME))
#export XSEG_HOME=$(shell ${XSEG_HOME})
#endif
ifeq (,$(XSEG_HOME))
export XSEG_HOME=$(CURDIR)
endif
CONFIG=./config.mk
#default:
#.PHONY: clean-config
#clean: clean-config
#clean-config:
# rm -f $(CONFIG)
ifndef BASE
exists=$(shell [ -f "$(CONFIG)" ] && echo exists)
ifeq (exists,$(exists))
include $(CONFIG)
else
$(shell $(XSEG_HOME)/envsetup show | sed -e 's/"//g' > "$(CONFIG)")
include $(CONFIG)
endif
ifeq (,$(XSEG_DOMAIN_TARGETS))
export XSEG_DOMAIN_TARGETS=$(shell $(XSEG_HOME)/tools/xseg-domain-targets | sed -e 's/^[^=]*=//;s/"//g')
endif
export BASE=$(XSEG_HOME)
endif
# Host configuration
XSEG_MOPTS="-march=core2"
# Order is significant due to dependencies
# (e.g. 'peers' should be last).
XSEG_DIRS="tools xtypes xseg sys drivers peers"
#XSEG_DOMAIN_DIRS="util_libs sys drivers peers"
# Host configuration
MOPTS=-march=core2
# Order is significant due to dependencies.
# 'peers' should be last.
MAKEDIRS=xq drivers xseg peers sys
Archipelagos Components
=======================
This document intends to document the various components of Archipelagos, the
storage infrastructure behind the Cydlades IaaS cloud software.
xq
**
Fixed-size (power of 2, up to 2^32), double-ended, circular queue of xserial
(unsigned int) elements, which guarantees safe concurrent access (with built-in
locking).
xq is used to safely index and/or split a shared buffer.
xseg
****
The backbone of Archipelagos.
The xseg API (and its implementation) provides access to shared memory segments,
residing either in userspace (posix segments) or in kernel space (segdev
segments), to both user- and kernel-space 'processes/threads' (known as peers,
in the xseg API).
The shared memory segments (referred to as plainly segments in the xseg API) are
formed by a pool of requests (xseg requests), and multiple endpoints (xseg
ports), on which the xseg peers bind.
The xseg API defines certain operations that can be performed on a segment
(malloc / allocate / map / etc), and ceration operations that can performed by
peers (signal / wait_for_signal / prepare_wait / cancel_wait etc).
Intented audience
=================
People who want to tinker with xseg internals, and need a quick-n-dirty intro
to the basics behind its operation.
Terminology / API
*****************
Client, server, peer, xsegbd, vlmc, map, request, segment, signal, wait...
Basic flow
**********
The initiator/client... then...
* Load the modules, mknod, create segment
- insmod lib/kernel/xseg.ko
- insmod lib/kernel/segdev.ko
- mknod /dev/segdev c 60 0
- insmod lib/kernel/xseg_segdev.ko
- insmod lib/kernel/xseg_posix.ko
- ./peer/users/xseg segdev:xsegbd:16:4096:64:1024:12 create
- insmod lib/kernel/xsegbd.ko
* Verify that the segment was created successfully:
- ./peer/users/xseg segdev:xsegbd reportall
* Allocate request for vlmcd:
- ./peer/users/xseg segdev:xsegbd 0:1 alloc_requests 1024
* Run vlmcd/mapperd/filed:
- ./peer/users/filed /srv/archip/images -p 9 -g segdev:xsegbd
- ./peer/users/mapperd[.py] -p 10 -g segdev:xsegbd
- ./peer/users/vlmcd -p 0 -g segdev:xsegbd -b 9 -m 10
* Create/List/Map/Unmap/Remove a volume:
- vlmc create test -s 4096
- vlmc list
- echo "test 1:0:1024" > /sys/bus/xsegbd/add
(vlmc map test might work as long as vlmcd runs on port 0, and port 1-2-n, where n the no of volumes mapped are free)
- vlmc showmapped
- echo 0 > /sys/bus/xsegbd/remove (or vlmc unmap /dev/xsegbd0)
- vlmc rm test
.PHONY: all clean xseg lib
.PHONY: install-src
BASE=..
include $(XSEG_HOME)/base.mk
include $(BASE)/config.mk
include $(BASE)/base.mk
FILES="Makefile"
FILES+=$(shell ls *.h)
DRIVERS=xseg_posix xseg_xsegdev
DRVOBJS=$(DRIVERS:=.o)
DRVSOS=$(DRIVERS:=.so)
SUBDIR:=$(subst $(XSEG_HOME)/,,$(CURDIR))
XSEG_CLEAN_TARGETS=$(patsubst %,clean-%,$(XSEG_DOMAIN_TARGETS))
XSEG_INSTALL_TARGETS=$(patsubst %,install-%,$(XSEG_DOMAIN_TARGETS))
XSEG_INSTALLSRC_TARGETS=$(patsubst %,install-src-%,$(XSEG_DOMAIN_TARGETS))
all: xseg $(DRIVERS:=.so) lib
$(BASE)/sys/xseg_user.o:
make -C $(BASE)/sys xseg_user.o
default: $(XSEG_DOMAIN_TARGETS)
xseg_posix.o: xseg_posix.c $(BASE)/xseg/xseg.h
$(CC) $(CFLAGS) -fPIC -c -o $@ $<
install: $(XSEG_INSTALL_TARGETS)
xseg_posix.so: xseg_posix.o $(BASE)/sys/xseg_user.o
$(CC) -shared -o $@ $< $(BASE)/sys/xseg_user.o
clean: $(XSEG_CLEAN_TARGETS)
xseg_xsegdev.o: xseg_xsegdev.c $(BASE)/xseg/xseg.h
$(CC) $(CFLAGS) -I$(BASE)/sys -fPIC -c -o $@ $<
.PHONY: $(XSEG_DOMAIN_TARGETS)
$(XSEG_DOMAIN_TARGETS):
@echo " ";
@echo "=== $@ in $(SUBDIR) ===";
@echo " ";
@make -C $@ TARGET=$@ default || exit 1;
@echo " ";
xseg_xsegdev.so: xseg_xsegdev.o $(BASE)/sys/xseg_user.o
$(CC) -shared -o $@ $< $(BASE)/sys/xseg_user.o
.PHONY: $(XSEG_INSTALL_TARGETS)
$(XSEG_INSTALL_TARGETS):
@echo " ";
@echo "=== $@ in $(SUBDIR)===";
@echo " ";
@make -C $(@:install-%=%) TARGET=$@ install || true;
@echo " ";
.PHONY: $(XSEG_INSTALLSRC_TARGETS)
$(XSEG_INSTALLSRC_TARGETS):
@echo " ";
@echo "=== $@ in $(SUBDIR)===";
@echo " ";
@make -C $(@:install-src-%=%) TARGET=$@ install-src || true;
@echo " ";
.PHONY: $(XSEG_CLEAN_TARGETS)
$(XSEG_CLEAN_TARGETS):
@echo " ";
@echo "=== $@ in $(SUBDIR)===";
@echo " ";
@make -C $(@:clean-%=%) TARGET=$@ clean || true;
@echo " ";
install-src: $(XSEG_INSTALLSRC_TARGETS)
@install -d $(DESTDIR)$(srcdir)$(SUBDIR) ;
@for f in $(FILES) ; do \
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir)$(SUBDIR) $$f ; \
done
lib:
cp -vaf $(DRVSOS) ../lib
clean:
rm -f $(DRVOBJS) $(DRVSOS)
# Copyright 2012 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above
# copyright notice, this list of conditions and the following
# disclaimer.
# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
#
.PHONY: default all clean install install-src
include $(XSEG_HOME)/base.mk
ifeq (,$(KVER))
KVER := $(shell uname -r)
endif
KDIR := /lib/modules/$(KVER)/build
PWD := $(shell pwd)
EXTRA_CFLAGS += -g -I$(BASE) -I$(BASE)/sys/kernel
LIBDIR=$(BASE)/lib/kernel
obj-m += xseg_segdev.o
obj-m += xseg_posix.o
obj-m += xseg_pthread.o
FILES="Makefile"
#FILES+=$(shell ls *.h)
FILES+=$(shell ls *.c)
SUBDIR:=$(subst $(XSEG_HOME),,$(CURDIR))
default: all
$(BASE)/sys/kernel/Module.symvers:
$(MAKE) -C $(BASE)/sys/kernel TARGET=kernel default
all: $(BASE)/sys/kernel/Module.symvers
cp -vaf $< .
$(MAKE) -C $(KDIR) SUBDIRS=$(PWD) V=$(V) modules
cp -vaf xseg_pthread.ko xseg_segdev.ko xseg_posix.ko $(LIBDIR)
install:
install -d $(DESTDIR)$(moduledir)
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(moduledir) xseg_segdev.ko
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(moduledir) xseg_posix.ko
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(moduledir) xseg_pthread.ko
install-src:
install -d $(DESTDIR)$(srcdir)$(SUBDIR) ;
@for f in $(FILES) ; do \
install -o 0 -g 0 -m 644 -t $(DESTDIR)$(srcdir)$(SUBDIR) $$f ; \
done
clean:
rm -f Module.symvers
make -C /lib/modules/$(KVER)/build M=$(PWD) V=$(V) clean
/*
* Copyright (C) 2012 GRNET S.A.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/* xseg_posix.c
* kernel driver for posix peers
*/
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/timer.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/genhd.h>
#include <linux/blkdev.h>
#include <linux/bio.h>
#include <linux/device.h>
#include <linux/completion.h>
#include <xseg/xseg.h>
#include <sys/kernel/segdev.h>
#include <sys/util.h>
#include <drivers/xseg_posix.h>
MODULE_DESCRIPTION("xseg_posix");
MODULE_AUTHOR("XSEG");
MODULE_LICENSE("GPL");
static int posix_remote_signal_init(void)
{
return 0;
}
static void posix_remote_signal_quit(void)
{
return;
}
static int posix_local_signal_init(struct xseg *xseg, xport portno)
{
return -1;
}
static void posix_local_signal_quit(struct xseg *xseg, xport portno)
{
return;
}
static int posix_prepare_wait(struct xseg *xseg, uint32_t portno)
{
return -1;
}
static int posix_cancel_wait(struct xseg *xseg, uint32_t portno)
{
return -1;
}
static int posix_wait_signal(struct xseg *xseg, uint32_t timeout)
{
return -1;
}
static int posix_signal(struct xseg *xseg, uint32_t portno)
{
struct pid *pid;
struct task_struct *task;
int ret = -ENOENT;
uint64_t p;
struct xseg_port *port = xseg_get_port(xseg, portno);
if (!port)
return -1;
struct posix_signal_desc *psd = xseg_get_signal_desc(xseg, port);
if (!psd)
return -1;
rcu_read_lock();
/* XXX Security: xseg peers can kill anyone */
p = * (volatile uint64_t *) &psd->waitcue;
if (!p) {
ret = 0;
goto out;
}
pid = find_vpid((pid_t)p);
if (!pid)
goto out;
task = pid_task(pid, PIDTYPE_PID);
if (!task)
goto out;
ret = send_sig(SIGIO, task, 1);
out:
rcu_read_unlock();
return ret;
}
static void *posix_malloc(uint64_t size)
{
return NULL;
}
static void *posix_realloc(void *mem, uint64_t size)
{
return NULL;
}
static void posix_mfree(void *mem) { }
int posix_init_signal_desc(struct xseg *xseg, void *sd)
{
return -1;
}
void posix_quit_signal_desc(struct xseg *xseg, void *sd)
{
return;
}
void * posix_alloc_data(struct xseg *xseg)
{
return NULL;
}
void posix_free_data(struct xseg *xseg, void *data)
{
return;
}
void *posix_alloc_signal_desc(struct xseg *xseg, void *data)
{
return NULL;
}
void posix_free_signal_desc(struct xseg *xseg, void *data, void *sd)
{
return;
}
static struct xseg_peer xseg_peer_posix = {
/* xseg signal operations */
{
.init_signal_desc = posix_init_signal_desc,
.quit_signal_desc = posix_quit_signal_desc,
.alloc_data = posix_alloc_data,
.free_data = posix_free_data,
.alloc_signal_desc = posix_alloc_signal_desc,
.free_signal_desc = posix_free_signal_desc,
.local_signal_init = posix_local_signal_init,
.local_signal_quit = posix_local_signal_quit,
.remote_signal_init = posix_remote_signal_init,
.remote_signal_quit = posix_remote_signal_quit,
.prepare_wait = posix_prepare_wait,
.cancel_wait = posix_cancel_wait,
.prepare_wait = posix_prepare_wait,
.wait_signal = posix_wait_signal,
.signal = posix_signal,
.malloc = posix_malloc,
.realloc = posix_realloc,
.mfree = posix_mfree
},
/* name */
"posix"
};
static int posix_init(void)
{
int r;