Changed build system to CMake

1 files changed, 230 insertions, 0 deletions

diff --git a/src/kiro-rdma.h b/src/kiro-rdma.h
new file mode 100644
index 0000000..fa16fd1
--- /dev/null
+++ b/src/kiro-rdma.h
@@ -0,0 +1,230 @@
+/* Copyright (C) 2014 Timo Dritschler <timo.dritschler@kit.edu>
+   (Karlsruhe Institute of Technology)
+
+   This library is free software; you can redistribute it and/or modify it
+   under the terms of the GNU Lesser General Public License as published by the
+   Free Software Foundation; either version 2.1 of the License, or (at your
+   option) any later version.
+
+   This library is distributed in the hope that it will be useful, but WITHOUT
+   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+   details.
+
+   You should have received a copy of the GNU Lesser General Public License along
+   with this library; if not, write to the Free Software Foundation, Inc., 51
+   Franklin St, Fifth Floor, Boston, MA 02110, USA
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#ifndef __KIRO_RDMA_H__
+#define __KIRO_RDMA_H__
+
+
+#include <rdma/rdma_cma.h>
+
+  
+struct kiro_connection_context {
+        
+    // Information and necessary structurs
+    uint32_t                identifier;             // Unique Identifier for this connection (Application Specific)
+    struct kiro_rdma_mem    *cf_mr_recv;            // Control-Flow Memory Region Receive
+    struct kiro_rdma_mem    *cf_mr_send;            // Control-Flow Memory Region Send
+    struct kiro_rdma_mem    *rdma_mr;               // Memory Region for RDMA Operations
+
+    struct ibv_mr           peer_mr;                // RDMA Memory Region Information of the peer
+    
+    enum {
+        KIRO_IDLE,
+        KIRO_MRI_REQUESTED,                         // Memory Region Information Requested
+        KIRO_RDMA_ESTABLISHED,                      // MRI Exchange complete. RDMA is ready
+        KIRO_RDMA_ACTIVE                            // RDMA Operation is being performed
+    } rdma_state;
+    
+};
+
+
+struct kiro_connection {
+    
+    uint32_t                identifier;
+    struct rdma_cm_id       *id;
+    struct kiro_connection  *next;
+    
+};
+
+
+struct kiro_ctrl_msg {
+    
+    enum {
+        KIRO_REQ_RDMA,                              // Requesting RDMA Access to/from the peer
+        KIRO_ACK_RDMA,                              // acknowledge RDMA Request and provide Memory Region Information
+        KIRO_REJ_RDMA                               // RDMA Request rejected :(  (peer_mri will be invalid)
+    } msg_type;
+    
+    struct ibv_mr peer_mri;
+    
+};
+
+
+struct kiro_rdma_mem {
+    
+    void            *mem;   // Pointer to the beginning of the memory block           
+    struct ibv_mr   *mr;    // Memory Region associated with the memory
+    size_t          size;   // Size in Bytes of the memory block
+
+};
+
+
+static int kiro_attach_qp (struct rdma_cm_id *id)
+{
+    if(!id)
+        return -1;
+    
+    id->pd = ibv_alloc_pd(id->verbs);
+    id->send_cq_channel = ibv_create_comp_channel(id->verbs);
+    id->recv_cq_channel = id->send_cq_channel; //we use one shared completion channel
+    id->send_cq = ibv_create_cq(id->verbs, 1, id, id->send_cq_channel, 0);
+    id->recv_cq = id->send_cq; //we use one shared completion queue
+    
+    struct ibv_qp_init_attr qp_attr;
+    memset(&qp_attr, 0, sizeof(struct ibv_qp_init_attr));
+    qp_attr.qp_context = (uintptr_t)id;
+    qp_attr.send_cq = id->send_cq;
+    qp_attr.recv_cq = id->recv_cq;
+    qp_attr.qp_type = IBV_QPT_RC;
+    qp_attr.cap.max_send_wr = 1;
+    qp_attr.cap.max_recv_wr = 1;
+    qp_attr.cap.max_send_sge = 1;
+    qp_attr.cap.max_recv_sge = 1;
+    
+    return rdma_create_qp(id, id->pd, &qp_attr);
+}
+
+
+static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr **mr, void *mem, size_t mem_size, int access)
+{
+    
+    if(mem_size == 0)
+    {    
+        printf("Cant allocate memory of size '0'.\n");
+        return -1;
+    }
+    
+    void *mem_handle = mem;
+    
+    if(!mem_handle)
+        mem_handle = malloc(mem_size);
+        
+    if(!mem_handle)
+    {
+        printf("Failed to allocate memory [Register Memory].");
+        return -1;
+    }        
+    
+    *mr = ibv_reg_mr(pd, mem_handle, mem_size, access);
+    if(!(*mr))
+    {
+        // Memory Registration failed
+        printf("Failed to register memory region!\n");
+        free(mem_handle);
+        return -1;
+    }
+    
+    return 0;
+}
+
+
+static struct kiro_rdma_mem* kiro_create_rdma_memory (struct ibv_pd *pd, size_t mem_size, int access)
+{
+    if(mem_size == 0)
+    {    
+        printf("Cant allocate memory of size '0'.\n");
+        return NULL;
+    }
+    
+    struct kiro_rdma_mem *krm = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem));
+    if(!krm)
+    {
+        printf("Failed to create new KIRO RDMA Memory.\n");
+        return NULL;
+    }
+    
+    if(kiro_register_rdma_memory(pd, &(krm->mr), krm->mem, mem_size, access))
+    {
+        free(krm);
+        return NULL;
+    }
+    
+    if(!krm->mem)
+        krm->mem = krm->mr->addr;
+    
+    
+    return krm;
+    
+}
+
+
+static void kiro_destroy_rdma_memory (struct kiro_rdma_mem *krm)
+{
+    if(!krm)
+        return;
+        
+    if(krm->mr)
+        ibv_dereg_mr(krm->mr);
+        
+    if(krm->mem)
+        free(krm->mem);
+        
+    free(krm);
+    krm = NULL;
+}
+
+
+static void kiro_destroy_connection_context (struct kiro_connection_context **ctx)
+{
+    if(!ctx)
+        return;
+    
+    if(!(*ctx))
+        return;
+        
+    if((*ctx)->cf_mr_recv)
+        kiro_destroy_rdma_memory((*ctx)->cf_mr_recv);
+    if((*ctx)->cf_mr_send)
+        kiro_destroy_rdma_memory((*ctx)->cf_mr_send);
+        
+    //The RDMA-Memory Region normally contains allocated memory from the USER that has
+    //just been 'registered' for RDMA. DON'T free it! Just deregister it. The user is
+    //responsible for freeing this memory. 
+    if((*ctx)->rdma_mr)
+    {
+        if((*ctx)->rdma_mr->mr)
+            ibv_dereg_mr((*ctx)->rdma_mr->mr);
+           
+        free((*ctx)->rdma_mr);
+        (*ctx)->rdma_mr = NULL;
+    }
+
+    free(*ctx);
+    *ctx = NULL;
+}
+
+
+static void kiro_destroy_connection (struct rdma_cm_id **conn)
+{
+    if(!(*conn))
+        return;
+        
+    rdma_disconnect(*conn);
+    struct kiro_connection_context *ctx = (struct kiro_connection_context *)((*conn)->context);
+    if(ctx)
+        kiro_destroy_connection_context(&ctx);
+        
+    rdma_destroy_ep(*conn);
+    *conn = NULL;
+}
+
+
+#endif //__KIRO_RDMA_H__   
\ No newline at end of file