KIRO Server and Client now automatically exchange MRI on connect

Added 'kiro_client_sync' that RDMA-READs the KIRO-TRB from the server
Updated Makefile
Updated test-client

6 files changed, 138 insertions, 33 deletions

diff --git a/Makefile b/Makefile
index 4e39036..f0f272b 100644
--- a/Makefile
+++ b/Makefile
@@ -24,11 +24,11 @@ test: test-trb client server
 test-trb: kiro-trb.o test.c
 	$(CC) $(CFLAGS) $(LDFLAGS) test.c kiro-trb.o -o test-trb
 
-client: kiro-client.o test-client.c
-	$(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o -o client
+client: kiro-client.o kiro-trb.o test-client.c
+	$(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o kiro-trb.o -o client
 
-server: kiro-server.o test-server.c
-	$(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o -o server
+server: kiro-server.o kiro-trb.o test-server.c
+	$(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o kiro-trb.o -o server
 
 
 .PHONY : clean
diff --git a/kiro-client.c b/kiro-client.c
index 41f4ff5..253251f 100644
--- a/kiro-client.c
+++ b/kiro-client.c
@@ -33,6 +33,7 @@
 #include <glib.h>
 #include "kiro-client.h"
 #include "kiro-rdma.h"
+#include "kiro-trb.h"
 
 #include <errno.h>
 
@@ -50,8 +51,9 @@ struct _KiroClientPrivate {
 
     /* 'Real' private structures */
     /* (Not accessible by properties) */
-    struct rdma_event_channel   *ec;    // Main Event Channel
-    struct rdma_cm_id           *conn;  // Connection to the Server
+    struct rdma_event_channel   *ec;        // Main Event Channel
+    struct rdma_cm_id           *conn;      // Connection to the Server
+    KiroTrb                     *buffer;    // Ring Buffer used to hold data from server
     
 
 };
@@ -64,12 +66,15 @@ static void kiro_client_init (KiroClient *self)
 {
     KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self);
     memset(priv, 0, sizeof(&priv));
+    priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL);
 }
 
 static void
 kiro_client_finalize (GObject *object)
 {
-    //PASS
+    KiroClient *self = KIRO_CLIENT(object);
+    KiroClientPrivate * priv = KIRO_CLIENT_GET_PRIVATE(self);
+    g_object_unref(priv->buffer);
 }
 
 static void
@@ -162,38 +167,74 @@ int kiro_client_connect (KiroClient *self, char *address, char* port)
         rdma_destroy_ep(priv->conn);
         return -1;
     }
+    printf("Connected to server.\n");
     
-    priv->ec = rdma_create_event_channel();
-    int oldflags = fcntl (priv->ec->fd, F_GETFL, 0);
-    /* Only change the FD Mode if we were able to get its flags */
-    if (oldflags >= 0) {
-        oldflags |= O_NONBLOCK;
-        /* Store modified flag word in the descriptor. */
-        fcntl (priv->ec->fd, F_SETFL, oldflags);
-    }
-    if(rdma_migrate_id(priv->conn, priv->ec))
+    
+    struct ibv_wc wc;
+    if(rdma_get_recv_comp(priv->conn, &wc) < 0)
     {
-        printf("Was unable to migrate connection to new Event Channel.\n");
+        printf("Failure waiting for POST from server.\n");
         rdma_disconnect(priv->conn);
         kiro_destroy_connection_context(ctx);
         rdma_destroy_ep(priv->conn);
         return -1;
     }
+    printf("Got Message from Server.\n");
+    ctx->peer_mr = (((struct kiro_ctrl_msg *)(ctx->cf_mr_recv->mem))->peer_mri);
+    printf("Expected TRB Size is: %u\n",ctx->peer_mr.length);
     
-    //ToDo:
-    //Create TRB, request RDMA from Server, call kiro_client_sync, ???, Profit!
-    
+    ctx->rdma_mr = kiro_create_rdma_memory(priv->conn->pd, ctx->peer_mr.length, IBV_ACCESS_LOCAL_WRITE);
+    if(!ctx->rdma_mr)
+    {
+        printf("Failed to allocate memory for receive buffer.\n");
+        rdma_disconnect(priv->conn);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->conn);
+        return -1;
+    }
+    printf("Connection setup completed successfully!\n");
     
-    printf("Connected to server.\n");
     return 0;
-    
 }
 
 
 
 int kiro_client_sync (KiroClient *self)
-{
-    //PASS
+{   
+    KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self);
+    struct kiro_connection_context *ctx = (struct kiro_connection_context *)priv->conn->context;
+    
+    if(rdma_post_read(priv->conn, priv->conn, ctx->rdma_mr->mem, ctx->peer_mr.length, ctx->rdma_mr->mr, 0, ctx->peer_mr.addr, ctx->peer_mr.rkey))
+    {
+        printf("Failed to read from server.\n");
+        rdma_disconnect(priv->conn);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->conn);
+        return -1;
+    }
+    
+    struct ibv_wc wc;
+    if(rdma_get_send_comp(priv->conn, &wc) < 0)
+    {
+        printf("Failure reading from server.\n");
+        rdma_disconnect(priv->conn);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->conn);
+        return -1;
+    }
+    
+    if(!kiro_trb_is_setup(priv->buffer))
+    {
+        //First time setup
+        kiro_trb_ingest(priv->buffer, ctx->rdma_mr->mem);
+    }
+    else
+    {
+        //Refresh
+        kiro_trb_refresh(priv->buffer);
+    }
+    
+    printf("Buffer successfully read from server.\n");
     return 0;
 }
 
diff --git a/kiro-client.h b/kiro-client.h
index 0f6881c..61e5676 100644
--- a/kiro-client.h
+++ b/kiro-client.h
@@ -77,9 +77,9 @@ GObject     kiro_client_new                 (void);
 /* client functions */
 
 
-int         kiro_client_connect             (KiroClient *, char *, char*);
+int         kiro_client_connect             (KiroClient* client, char* dest_addr, char* dest_port);
 
-int         kiro_client_sync                (KiroClient *);
+int         kiro_client_sync                (KiroClient* client);
 
 
 G_END_DECLS
diff --git a/kiro-server.c b/kiro-server.c
index 119f0c1..d4cf7bc 100644
--- a/kiro-server.c
+++ b/kiro-server.c
@@ -33,6 +33,7 @@
 #include <glib.h>
 #include "kiro-server.h"
 #include "kiro-rdma.h"
+#include "kiro-trb.h"
 
 
 /*
@@ -51,6 +52,7 @@ struct _KiroServerPrivate {
     struct rdma_event_channel   *ec;        // Main Event Channel
     struct rdma_cm_id           *base;      // Base-Listening-Connection
     struct kiro_connection      *client;    // Connection to the client
+    KiroTrb                     *buffer;    // Memory Container
     
 
 };
@@ -63,12 +65,21 @@ static void kiro_server_init (KiroServer *self)
 {
     KiroServerPrivate *priv = KIRO_SERVER_GET_PRIVATE(self);
     memset(priv, 0, sizeof(&priv));
+    
+    priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL);
+    kiro_trb_reshape(priv->buffer, sizeof(uint64_t), 1000);
+    uint64_t a = 0xAFFED00F;
+    uint64_t b = 0x1337BEEF;
+    kiro_trb_push(priv->buffer, &a);
+    kiro_trb_push(priv->buffer, &b);
 }
 
 static void
 kiro_server_finalize (GObject *object)
 {
-    //PASS
+    KiroServer *self = KIRO_SERVER(object);
+    KiroServerPrivate * priv = KIRO_SERVER_GET_PRIVATE(self);
+    g_object_unref(priv->buffer);
 }
 
 static void
@@ -218,13 +229,65 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
         return -1;
     }
     
-    //ToDo:
-    //Create TRB, request RDMA from Server, call kiro_server_sync, ???, Profit!
+    printf("Client Connected.\n");
     
+        
+    ctx->rdma_mr = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem));
+    if(!ctx->rdma_mr)
+    {
+        printf("Failed to allocate RDMA Memory Container.\n");
+        rdma_disconnect(priv->client->id);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->base);
+        rdma_destroy_ep(priv->client->id);
+        free(priv->client);
+        return -1;
+    }
     
-    printf("Client Connected.\n");
-    return 0;
+    ctx->rdma_mr->mem = kiro_trb_get_raw_buffer(priv->buffer);
+    ctx->rdma_mr->size = kiro_trb_get_raw_size(priv->buffer);
+    ctx->rdma_mr->mr = rdma_reg_read(priv->client->id, ctx->rdma_mr->mem, ctx->rdma_mr->size);
+    if(!ctx->rdma_mr->mr)
+    {
+        printf("Failed to register RDMA Memory Region.\n");
+        rdma_disconnect(priv->client->id);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->base);
+        rdma_destroy_ep(priv->client->id);
+        free(priv->client);
+        return -1;
+    }
     
+    struct kiro_ctrl_msg *msg = (struct kiro_ctrl_msg *)(ctx->cf_mr_send->mem);
+    msg->msg_type = KIRO_ACK_RDMA;
+    msg->peer_mri = *(ctx->rdma_mr->mr);
+    
+    if(rdma_post_send(priv->client->id, priv->client, ctx->cf_mr_send->mem, ctx->cf_mr_send->size, ctx->cf_mr_send->mr, IBV_SEND_SIGNALED))
+    {
+        printf("Failure while trying to post SEND.\n");
+        rdma_disconnect(priv->client->id);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->base);
+        rdma_destroy_ep(priv->client->id);
+        free(priv->client);
+        return -1;
+    }
+    
+    struct ibv_wc wc;
+    
+    if(rdma_get_send_comp(priv->client->id, &wc) < 0)
+    {
+        printf("Failed to post RDMA MRI to client.\n");
+        rdma_disconnect(priv->client->id);
+        kiro_destroy_connection_context(ctx);
+        rdma_destroy_ep(priv->base);
+        rdma_destroy_ep(priv->client->id);
+        free(priv->client);
+        return -1;
+    }
+    printf("RDMA MRI sent to client.\n");
+    sleep(1);
+    return 0;
 }
 
 
diff --git a/kiro-server.h b/kiro-server.h
index 6e7973c..1d85419 100644
--- a/kiro-server.h
+++ b/kiro-server.h
@@ -76,7 +76,7 @@ GObject     kiro_server_new                 (void);
 
 /* server functions */
 
-int         kiro_server_start               (KiroServer*, char*, char*);
+int         kiro_server_start               (KiroServer* server, char* bind_addr, char* bind_port);
 
 G_END_DECLS
 
diff --git a/test-client.c b/test-client.c
index 67dacf7..0a318f1 100644
--- a/test-client.c
+++ b/test-client.c
@@ -7,6 +7,7 @@
 int main(void)
 {
     KiroClient *client = g_object_new(KIRO_TYPE_CLIENT, NULL);
-    kiro_client_connect(client, "192.168.11.61", "60010", 5000);
+    kiro_client_connect(client, "192.168.11.61", "60010");
+    kiro_client_sync(client);
     return 0; 
 }
\ No newline at end of file