From 3405180e97cd6b4d4bef6fed2a7e666eb8126906 Mon Sep 17 00:00:00 2001 From: Timo Dritschler Date: Fri, 25 Apr 2014 19:32:51 +0200 Subject: KIRO Server and Client now automatically exchange MRI on connect Added 'kiro_client_sync' that RDMA-READs the KIRO-TRB from the server Updated Makefile Updated test-client --- Makefile | 8 +++--- kiro-client.c | 81 ++++++++++++++++++++++++++++++++++++++++++++--------------- kiro-client.h | 4 +-- kiro-server.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++---- kiro-server.h | 2 +- test-client.c | 3 ++- 6 files changed, 138 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 4e39036..f0f272b 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,11 @@ test: test-trb client server test-trb: kiro-trb.o test.c $(CC) $(CFLAGS) $(LDFLAGS) test.c kiro-trb.o -o test-trb -client: kiro-client.o test-client.c - $(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o -o client +client: kiro-client.o kiro-trb.o test-client.c + $(CC) $(CFLAGS) $(LDFLAGS) test-client.c kiro-client.o kiro-trb.o -o client -server: kiro-server.o test-server.c - $(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o -o server +server: kiro-server.o kiro-trb.o test-server.c + $(CC) $(CFLAGS) $(LDFLAGS) test-server.c kiro-server.o kiro-trb.o -o server .PHONY : clean diff --git a/kiro-client.c b/kiro-client.c index 41f4ff5..253251f 100644 --- a/kiro-client.c +++ b/kiro-client.c @@ -33,6 +33,7 @@ #include #include "kiro-client.h" #include "kiro-rdma.h" +#include "kiro-trb.h" #include @@ -50,8 +51,9 @@ struct _KiroClientPrivate { /* 'Real' private structures */ /* (Not accessible by properties) */ - struct rdma_event_channel *ec; // Main Event Channel - struct rdma_cm_id *conn; // Connection to the Server + struct rdma_event_channel *ec; // Main Event Channel + struct rdma_cm_id *conn; // Connection to the Server + KiroTrb *buffer; // Ring Buffer used to hold data from server }; @@ -64,12 +66,15 @@ static void kiro_client_init (KiroClient *self) { KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self); memset(priv, 0, sizeof(&priv)); + priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL); } static void kiro_client_finalize (GObject *object) { - //PASS + KiroClient *self = KIRO_CLIENT(object); + KiroClientPrivate * priv = KIRO_CLIENT_GET_PRIVATE(self); + g_object_unref(priv->buffer); } static void @@ -162,38 +167,74 @@ int kiro_client_connect (KiroClient *self, char *address, char* port) rdma_destroy_ep(priv->conn); return -1; } + printf("Connected to server.\n"); - priv->ec = rdma_create_event_channel(); - int oldflags = fcntl (priv->ec->fd, F_GETFL, 0); - /* Only change the FD Mode if we were able to get its flags */ - if (oldflags >= 0) { - oldflags |= O_NONBLOCK; - /* Store modified flag word in the descriptor. */ - fcntl (priv->ec->fd, F_SETFL, oldflags); - } - if(rdma_migrate_id(priv->conn, priv->ec)) + + struct ibv_wc wc; + if(rdma_get_recv_comp(priv->conn, &wc) < 0) { - printf("Was unable to migrate connection to new Event Channel.\n"); + printf("Failure waiting for POST from server.\n"); rdma_disconnect(priv->conn); kiro_destroy_connection_context(ctx); rdma_destroy_ep(priv->conn); return -1; } + printf("Got Message from Server.\n"); + ctx->peer_mr = (((struct kiro_ctrl_msg *)(ctx->cf_mr_recv->mem))->peer_mri); + printf("Expected TRB Size is: %u\n",ctx->peer_mr.length); - //ToDo: - //Create TRB, request RDMA from Server, call kiro_client_sync, ???, Profit! - + ctx->rdma_mr = kiro_create_rdma_memory(priv->conn->pd, ctx->peer_mr.length, IBV_ACCESS_LOCAL_WRITE); + if(!ctx->rdma_mr) + { + printf("Failed to allocate memory for receive buffer.\n"); + rdma_disconnect(priv->conn); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->conn); + return -1; + } + printf("Connection setup completed successfully!\n"); - printf("Connected to server.\n"); return 0; - } int kiro_client_sync (KiroClient *self) -{ - //PASS +{ + KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self); + struct kiro_connection_context *ctx = (struct kiro_connection_context *)priv->conn->context; + + if(rdma_post_read(priv->conn, priv->conn, ctx->rdma_mr->mem, ctx->peer_mr.length, ctx->rdma_mr->mr, 0, ctx->peer_mr.addr, ctx->peer_mr.rkey)) + { + printf("Failed to read from server.\n"); + rdma_disconnect(priv->conn); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->conn); + return -1; + } + + struct ibv_wc wc; + if(rdma_get_send_comp(priv->conn, &wc) < 0) + { + printf("Failure reading from server.\n"); + rdma_disconnect(priv->conn); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->conn); + return -1; + } + + if(!kiro_trb_is_setup(priv->buffer)) + { + //First time setup + kiro_trb_ingest(priv->buffer, ctx->rdma_mr->mem); + } + else + { + //Refresh + kiro_trb_refresh(priv->buffer); + } + + printf("Buffer successfully read from server.\n"); return 0; } diff --git a/kiro-client.h b/kiro-client.h index 0f6881c..61e5676 100644 --- a/kiro-client.h +++ b/kiro-client.h @@ -77,9 +77,9 @@ GObject kiro_client_new (void); /* client functions */ -int kiro_client_connect (KiroClient *, char *, char*); +int kiro_client_connect (KiroClient* client, char* dest_addr, char* dest_port); -int kiro_client_sync (KiroClient *); +int kiro_client_sync (KiroClient* client); G_END_DECLS diff --git a/kiro-server.c b/kiro-server.c index 119f0c1..d4cf7bc 100644 --- a/kiro-server.c +++ b/kiro-server.c @@ -33,6 +33,7 @@ #include #include "kiro-server.h" #include "kiro-rdma.h" +#include "kiro-trb.h" /* @@ -51,6 +52,7 @@ struct _KiroServerPrivate { struct rdma_event_channel *ec; // Main Event Channel struct rdma_cm_id *base; // Base-Listening-Connection struct kiro_connection *client; // Connection to the client + KiroTrb *buffer; // Memory Container }; @@ -63,12 +65,21 @@ static void kiro_server_init (KiroServer *self) { KiroServerPrivate *priv = KIRO_SERVER_GET_PRIVATE(self); memset(priv, 0, sizeof(&priv)); + + priv->buffer = g_object_new(KIRO_TYPE_TRB, NULL); + kiro_trb_reshape(priv->buffer, sizeof(uint64_t), 1000); + uint64_t a = 0xAFFED00F; + uint64_t b = 0x1337BEEF; + kiro_trb_push(priv->buffer, &a); + kiro_trb_push(priv->buffer, &b); } static void kiro_server_finalize (GObject *object) { - //PASS + KiroServer *self = KIRO_SERVER(object); + KiroServerPrivate * priv = KIRO_SERVER_GET_PRIVATE(self); + g_object_unref(priv->buffer); } static void @@ -218,13 +229,65 @@ int kiro_server_start (KiroServer *self, char *address, char *port) return -1; } - //ToDo: - //Create TRB, request RDMA from Server, call kiro_server_sync, ???, Profit! + printf("Client Connected.\n"); + + ctx->rdma_mr = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem)); + if(!ctx->rdma_mr) + { + printf("Failed to allocate RDMA Memory Container.\n"); + rdma_disconnect(priv->client->id); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->base); + rdma_destroy_ep(priv->client->id); + free(priv->client); + return -1; + } - printf("Client Connected.\n"); - return 0; + ctx->rdma_mr->mem = kiro_trb_get_raw_buffer(priv->buffer); + ctx->rdma_mr->size = kiro_trb_get_raw_size(priv->buffer); + ctx->rdma_mr->mr = rdma_reg_read(priv->client->id, ctx->rdma_mr->mem, ctx->rdma_mr->size); + if(!ctx->rdma_mr->mr) + { + printf("Failed to register RDMA Memory Region.\n"); + rdma_disconnect(priv->client->id); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->base); + rdma_destroy_ep(priv->client->id); + free(priv->client); + return -1; + } + struct kiro_ctrl_msg *msg = (struct kiro_ctrl_msg *)(ctx->cf_mr_send->mem); + msg->msg_type = KIRO_ACK_RDMA; + msg->peer_mri = *(ctx->rdma_mr->mr); + + if(rdma_post_send(priv->client->id, priv->client, ctx->cf_mr_send->mem, ctx->cf_mr_send->size, ctx->cf_mr_send->mr, IBV_SEND_SIGNALED)) + { + printf("Failure while trying to post SEND.\n"); + rdma_disconnect(priv->client->id); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->base); + rdma_destroy_ep(priv->client->id); + free(priv->client); + return -1; + } + + struct ibv_wc wc; + + if(rdma_get_send_comp(priv->client->id, &wc) < 0) + { + printf("Failed to post RDMA MRI to client.\n"); + rdma_disconnect(priv->client->id); + kiro_destroy_connection_context(ctx); + rdma_destroy_ep(priv->base); + rdma_destroy_ep(priv->client->id); + free(priv->client); + return -1; + } + printf("RDMA MRI sent to client.\n"); + sleep(1); + return 0; } diff --git a/kiro-server.h b/kiro-server.h index 6e7973c..1d85419 100644 --- a/kiro-server.h +++ b/kiro-server.h @@ -76,7 +76,7 @@ GObject kiro_server_new (void); /* server functions */ -int kiro_server_start (KiroServer*, char*, char*); +int kiro_server_start (KiroServer* server, char* bind_addr, char* bind_port); G_END_DECLS diff --git a/test-client.c b/test-client.c index 67dacf7..0a318f1 100644 --- a/test-client.c +++ b/test-client.c @@ -7,6 +7,7 @@ int main(void) { KiroClient *client = g_object_new(KIRO_TYPE_CLIENT, NULL); - kiro_client_connect(client, "192.168.11.61", "60010", 5000); + kiro_client_connect(client, "192.168.11.61", "60010"); + kiro_client_sync(client); return 0; } \ No newline at end of file -- cgit v1.2.3