rdma helper More...

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <inttypes.h>
#include <endian.h>
#include <errno.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <pthread.h>
#include <semaphore.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <rdma/rdma_cma.h>
#include <netdb.h>
#include "utils.h"
#include "mooshika.h"

Macros
#define	EPOLL_MAX_EVENTS 16

#define	NUM_WQ_PER_POLL 16

#define	VALGRIND_MAKE_MEM_DEFINED(addr, len)

#define	THREAD_STACK_SIZE 2116488
	msk_create_thread: Simple wrapper around pthread_create More...

Functions
void	__attribute__ ((constructor))

void	__attribute__ ((destructor))

struct msk_pd *	msk_getpd (struct msk_trans *trans)
	msk_getpd: helper function to get the right pd for a given trans More...

struct ibv_mr *	msk_reg_mr (struct msk_trans trans, void memaddr, size_t size, int access)
	msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr) More...

int	msk_dereg_mr (struct ibv_mr *mr)
	msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr) More...

msk_rloc_t *	msk_make_rloc (struct ibv_mr *mr, uint64_t addr, uint32_t size)
	msk_make_rloc: makes a rkey to send it for remote host use More...

void	msk_print_devinfo (struct msk_trans *trans)

void *	msk_stats_thread (void *arg)
	msk_stats_thread: unix socket thread More...

void	msk_destroy_trans (struct msk_trans **ptrans)
	msk_destroy_trans: disconnects and free trans data More...

int	msk_init (struct msk_trans *ptrans, struct msk_trans_attr attr)
	msk_init: part of the init that's the same for client and server More...

int	msk_bind_server (struct msk_trans *trans)
	msk_bind_server More...

int	msk_finalize_accept (struct msk_trans *trans)
	msk_finalize_accept: does the real connection acceptance and wait for other side to be ready More...

struct msk_trans *	msk_accept_one_timedwait (struct msk_trans trans, struct timespec abstime)
	msk_accept_one: given a listening trans, waits till one connection is requested and accepts it More...

struct msk_trans *	msk_accept_one_wait (struct msk_trans *trans, int msleep)

int	msk_finalize_connect (struct msk_trans *trans)
	msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack More...

int	msk_connect (struct msk_trans *trans)
	msk_connect: connects a client to a server More...

int	msk_post_n_recv (struct msk_trans trans, msk_data_t data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
	msk_post_n_recv: Post a receive buffer. More...

int	msk_post_n_send (struct msk_trans trans, msk_data_t data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
	Post a send buffer. More...

int	msk_wait_n_recv (struct msk_trans trans, msk_data_t data, int num_sge)
	Post a receive buffer and waits for that one and not any other to be filled. More...

int	msk_wait_n_send (struct msk_trans trans, msk_data_t data, int num_sge)
	Post a send buffer and waits for that one to be completely sent. More...

int	msk_post_n_read (struct msk_trans trans, msk_data_t data, int num_sge, msk_rloc_t rloc, ctx_callback_t callback, ctx_callback_t err_callback, void callback_arg)

int	msk_post_n_write (struct msk_trans trans, msk_data_t data, int num_sge, msk_rloc_t rloc, ctx_callback_t callback, ctx_callback_t err_callback, void callback_arg)

int	msk_wait_n_read (struct msk_trans trans, msk_data_t data, int num_sge, msk_rloc_t *rloc)

int	msk_wait_n_write (struct msk_trans trans, msk_data_t data, int num_sge, msk_rloc_t *rloc)

struct sockaddr *	msk_get_dst_addr (struct msk_trans *trans)

struct sockaddr *	msk_get_src_addr (struct msk_trans *trans)

uint16_t	msk_get_src_port (struct msk_trans *trans)

uint16_t	msk_get_dst_port (struct msk_trans *trans)

const char *	msk_wc_status_str (enum ibv_wc_status status)

Detailed Description

rdma helper

This is (very) loosely based on a mix of diod, rping (librdmacm/examples) and kernel's net/9p/trans_rdma.c

Macro Definition Documentation

#define EPOLL_MAX_EVENTS 16

#define NUM_WQ_PER_POLL 16

#define THREAD_STACK_SIZE 2116488

msk_create_thread: Simple wrapper around pthread_create

#define VALGRIND_MAKE_MEM_DEFINED	(	addr,
		len
	)

Function Documentation

void __attribute__ ( (constructor) )

References ERROR_LOG.

void __attribute__ ( (destructor) )

struct msk_trans* msk_accept_one_timedwait	(	struct msk_trans *	trans,
		struct timespec *	abstime
	)

msk_accept_one: given a listening trans, waits till one connection is requested and accepts it

Parameters

trans [IN] the parent trans

Returns: a new trans for the child on success, NULL on failure

Todo:: make it return an int an' use trans as argument

Todo:: timeout?

References msk_trans::debug, INFO_LOG, and MSK_DEBUG_EVENT.

struct msk_trans* msk_accept_one_wait	(	struct msk_trans *	trans,
		int	msleep
	)

References NSEC_IN_SEC.

int msk_bind_server ( struct msk_trans * trans )

msk_bind_server

Parameters

trans [INOUT]

Returns: 0 on success, errno value on failure

Referenced by main().

int msk_connect ( struct msk_trans * trans )

msk_connect: connects a client to a server

Parameters

trans [INOUT] trans must be init first

Returns: 0 on success, the value of errno on error

Referenced by main().

int msk_dereg_mr ( struct ibv_mr * mr )

msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr)

Parameters

mr	[INOUT] the mr to deregister

Returns: 0 on success, errno value on failure

Referenced by handle_trans(), and main().

void msk_destroy_trans ( struct msk_trans ** ptrans )

msk_destroy_trans: disconnects and free trans data

Parameters

ptrans [INOUT] pointer to the trans to destroy

Todo:: check if it is init. if not should just return EINVAL but.. lock.__lock, cond.__lock might work.

References msk_trans::cm_cond, msk_trans::cm_id, msk_trans::cm_lock, msk_trans::debug, INFO_LOG, msk_trans::MSK_CLOSED, msk_trans::MSK_CLOSING, MSK_DEBUG_CM_LOCKS, MSK_DEBUG_SETUP, msk_trans::MSK_ERROR, msk_trans::MSK_LISTENING, and msk_trans::state.

Referenced by handle_trans(), and main().

int msk_finalize_accept ( struct msk_trans * trans )

msk_finalize_accept: does the real connection acceptance and wait for other side to be ready

Parameters

trans [IN]

Returns: 0 on success, the value of errno on error

Referenced by handle_trans(), and main().

int msk_finalize_connect ( struct msk_trans * trans )

msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack

Parameters

trans [IN]

Returns: 0 on success, errno value on failure

Referenced by handle_trans(), and main().

struct sockaddr* msk_get_dst_addr ( struct msk_trans * trans )

uint16_t msk_get_dst_port ( struct msk_trans * trans )

struct sockaddr* msk_get_src_addr ( struct msk_trans * trans )

uint16_t msk_get_src_port ( struct msk_trans * trans )

struct msk_pd* msk_getpd ( struct msk_trans * trans )

msk_getpd: helper function to get the right pd for a given trans

Parameters

trans [IN] the connection handle

Returns: NULL if nothing is available, next free pd if none fit, correct one if any match

References atomic_dec, atomic_postinc, msk_trans::cm_id, msk_pd::context, msk_trans::pd, PD_GUARD, and msk_pd::used.

Referenced by msk_reg_mr(), and setup_recv().

int msk_init	(	struct msk_trans **	ptrans,
		struct msk_trans_attr *	attr
	)

msk_init: part of the init that's the same for client and server

Parameters

ptrans	[INOUT]
attr	[IN] attributes to set parameters in ptrans. attr->addr must be set, others can be either 0 or sane values.

Returns: 0 on success, errno value on failure

References INFO_LOG, and MSK_DEBUG_EVENT.

Referenced by main().

msk_rloc_t* msk_make_rloc	(	struct ibv_mr *	mr,
		uint64_t	addr,
		uint32_t	size
	)

msk_make_rloc: makes a rkey to send it for remote host use

Parameters

mr	[IN] the mr in which the addr belongs
addr	[IN] the addr to give
size	[IN] the size to allow (hint)

Returns: a pointer to the rkey on success, NULL on failure.

References INFO_LOG, MSK_DEBUG_EVENT, msk_rloc::raddr, msk_rloc::rkey, and msk_rloc::size.

Referenced by main().

int msk_post_n_read	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		msk_rloc_t *	rloc,
		ctx_callback_t	callback,
		ctx_callback_t	err_callback,
		void *	callback_arg
	)

int msk_post_n_recv	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		ctx_callback_t	callback,
		ctx_callback_t	err_callback,
		void *	callback_arg
	)

msk_post_n_recv: Post a receive buffer.

Need to post recv buffers before the opposite side tries to send anything!

Parameters

trans	[IN]
data	[OUT] the data buffer to be filled with received data
num_sge	[IN] the number of elements in data to register
callback	[IN] function that'll be called when done
err_callback	[IN] function that'll be called on error
callback_arg	[IN] argument to give to the callback

Returns: 0 on success, the value of errno on error

Todo:: np_uerror(ret)

Referenced by main().

int msk_post_n_send	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		ctx_callback_t	callback,
		ctx_callback_t	err_callback,
		void *	callback_arg
	)

Post a send buffer.

Parameters

trans	[IN]
data	[IN] the data buffer to be sent
num_sge	[IN] the number of elements in data to send
callback	[IN] function that'll be called when done
err_callback	[IN] function that'll be called on error
callback_arg	[IN] argument to give to the callback

Returns: 0 on success, the value of errno on error

Referenced by main().

int msk_post_n_write	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		msk_rloc_t *	rloc,
		ctx_callback_t	callback,
		ctx_callback_t	err_callback,
		void *	callback_arg
	)

References msk_trans::debug, and MSK_DEBUG_CM_LOCKS.

void msk_print_devinfo ( struct msk_trans * trans )

References msk_trans::cm_id.

struct ibv_mr* msk_reg_mr	(	struct msk_trans *	trans,
		void *	memaddr,
		size_t	size,
		int	access
	)

msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr)

Parameters

trans	[IN]
memaddr	[IN] the address to register
size	[IN] the size of the area to register
access	[IN] the access to grants to the mr (e.g. IBV_ACCESS_LOCAL_WRITE)

Returns: a pointer to the mr if registered correctly or NULL on failure

References msk_getpd(), msk_pd::pd, and msk_pd::used.

Referenced by handle_trans(), main(), and post_recvs().

void* msk_stats_thread ( void * arg )

msk_stats_thread: unix socket thread

Well, a thread. arg = trans

References msk_trans::debug, INFO_LOG, MSK_DEBUG_EVENT, msk_stats::nsec_callback, msk_stats::nsec_compevent, NSEC_IN_SEC, msk_stats::rx_bytes, msk_stats::rx_err, msk_stats::rx_pkt, msk_trans::stats, msk_trans::stats_sock, msk_stats::tx_bytes, msk_stats::tx_err, and msk_stats::tx_pkt.

int msk_wait_n_read	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		msk_rloc_t *	rloc
	)

int msk_wait_n_recv	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge
	)

Post a receive buffer and waits for that one and not any other to be filled.

Generally a bad idea to use that one unless only that one is used.

Parameters

trans	[IN]
data	[OUT] the data buffer to be filled with the received data
num_sge	[IN] the number of elements in data to register

Returns: 0 on success, the value of errno on error

int msk_wait_n_send	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge
	)

Post a send buffer and waits for that one to be completely sent.

Parameters

trans	[IN]
data	[IN] the data to send
num_sge	[IN] the number of elements in data to send

Returns: 0 on success, the value of errno on error

int msk_wait_n_write	(	struct msk_trans *	trans,
		msk_data_t *	data,
		int	num_sge,
		msk_rloc_t *	rloc
	)

const char* msk_wc_status_str ( enum ibv_wc_status status )

Macros

Functions

Detailed Description

Macro Definition Documentation

Function Documentation