RLib
Notice
The latest version has been transferred to https://github.com/wxdwfc/rlibv2, which is now actively maintained.
Intro
RLib is a header-only library for easier use of RDMA using C++. Basically it is a set of wrappers of the interfaces of libibverbs,
yet it additionally handles many tedius things, such as establishing connections between RDMA QPs, and simplifies many configurations.
To use
#include "rdma_ctrl.hpp" is all you need.
Example
Usually very few lines of code are needed to use RDMA with RLib. Below is a snippet of using RLib to implement a simple pingpong application using one-sided RDMA primitive.
Server side
/**
* Note, RDMA usually uses some other communication method (e.g. TCP/IP) to exchange QP informations.
* RLib uses TCP for the pre-communication.
*/
int server_node_id = 1;
int tcp_port = 8888;
int client_port = 8000;
using namespace rdmaio;
RdmaCtrl *c = new RdmaCtrl(server_node_id,tcp_port);
RdmaCtrl::DevIdx idx {.dev_id = 0,.port_id = 1 }; // using the first RNIC's first port
c->open_thread_local_device(idx);
// register a buffer to the previous opened device, using id = 73
char *buffer = (char *)malloc(4096);
memset(buffer, 0, 4096);
RDMA_ASSERT(c->register_memory(73,buffer,4096,c->get_device()) == true);
char s[] = "hello world";
memcpy(buffer, s, strlen(s));
RCQP *qp = c->create_rc_qp(create_rc_idx(1,0),c->get_device(),c->get_local_mr(73));
// server also needs to "connect" clinet.
while(qp->connect(client_ip,client_port) != SUCC) {
usleep(2000);
}
while(true) {
// This is RDMA, server does not need to do anything :)
sleep(1);
}Client side
int client_node_id = 0;
int tcp_port = 8000;
int server_port = 8888;
using namespace rdmaio;
RdmaCtrl *c = new RdmaCtrl(client_node_id,tcp_port);
RdmaCtrl::DevIdx idx {.dev_id = 0,.port_id = 1 }; // using the first RNIC's first port
c->open_thread_local_device(idx);
// register a buffer to the previous opened device, using id = 73
char *buffer = (char *)malloc(4096);
RDMA_ASSERT(c->register_memory(73,buffer,4096,c->get_device()) == true);
// get remote server's memory information
MemoryAttr mr;
while(QP::get_remote_mr(server_ip,server_port,73,&mr) != SUCC) {
usleep(2000);
}
// create the RC qp to access remote server's memory, using the previous registered memory
RCQP *qp = c->create_rc_qp(create_rc_idx(1,0),c->get_device(),c->get_local_mr(73));
qp->bind_remote_mr(mr); // bind to the previous allocated mr
while(qp->connect(server_ip,server_port) != SUCC) {
usleep(2000);
}
// main pingpong loop
ibv_wc wc;
while(true) {
char *local_buf = buffer;
uint64_t address = 0;
int msg_len = 11; // length of "hello world"
// read an uint64_t from the server
auto rc = qp->post_send(IBV_WR_RDMA_READ,local_buf,msg_len,address,IBV_SEND_SIGNALED);
qp->poll_till_completion();
// then get the results, stored in the local_buffer
}
Acknowledgments
TODO