From d17929dce11210dad47036c55f0fa60cb224392f Mon Sep 17 00:00:00 2001 From: Timo Dritschler Date: Fri, 25 Apr 2014 15:38:51 +0200 Subject: Fixed kiro_create_rdma_memory not setting pointers correctly Fixed a bug in KIRO Server and Client that prevented preemtive "Post Recv" --- kiro-client.c | 17 ++++++++++------- kiro-client.h | 2 +- kiro-rdma.h | 12 ++++++++---- kiro-server.c | 7 ++++++- test-client.c | 2 +- test-server.c | 2 +- 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/kiro-client.c b/kiro-client.c index 9fa15ac..41f4ff5 100644 --- a/kiro-client.c +++ b/kiro-client.c @@ -34,6 +34,8 @@ #include "kiro-client.h" #include "kiro-rdma.h" +#include + /* * Definition of 'private' structures and members and macro to access them @@ -79,7 +81,7 @@ kiro_client_class_init (KiroClientClass *klass) -int kiro_client_connect (KiroClient *self, char *address, char* port, size_t timeout) +int kiro_client_connect (KiroClient *self, char *address, char* port) { KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self); @@ -94,9 +96,10 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim hints.ai_port_space = RDMA_PS_IB; if(rdma_getaddrinfo(address, port, &hints, &res_addrinfo)) { - printf("Failed to resolve Route to server %s:%s\n",address, port); + printf("Failed to contruct address information for %s:%s\n",address, port); return -1; } + printf("Address information created.\n"); struct ibv_qp_init_attr qp_attr; memset(&qp_attr, 0, sizeof(qp_attr)); @@ -107,12 +110,12 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim qp_attr.qp_context = priv->conn; qp_attr.sq_sig_all = 1; - if(rdma_create_ep(&(priv->conn), res_addrinfo, NULL, &qp_attr)) { - printf("Endpoint creation failed.\n"); + printf("Endpoint creation failed with error: %i\n", errno); return -1; } + printf("Route to server resolved.\n"); struct kiro_connection_context *ctx = (struct kiro_connection_context *)calloc(1,sizeof(struct kiro_connection_context)); if(!ctx) @@ -144,9 +147,9 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim ctx->cf_mr_recv->size = ctx->cf_mr_send->size = sizeof(struct kiro_ctrl_msg); priv->conn->context = ctx; - if(!rdma_post_recv(priv->conn, priv->conn, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr)) + if(rdma_post_recv(priv->conn, priv->conn, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr)) { - printf("Posting preemtive receive for connection failed.\n"); + printf("Posting preemtive receive for connection failed with error: %i\n", errno); kiro_destroy_connection_context(ctx); rdma_destroy_ep(priv->conn); return -1; @@ -181,7 +184,7 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim //Create TRB, request RDMA from Server, call kiro_client_sync, ???, Profit! - printf("Connect to server.\n"); + printf("Connected to server.\n"); return 0; } diff --git a/kiro-client.h b/kiro-client.h index fb1dccb..0f6881c 100644 --- a/kiro-client.h +++ b/kiro-client.h @@ -77,7 +77,7 @@ GObject kiro_client_new (void); /* client functions */ -int kiro_client_connect (KiroClient *, char *, char*, size_t); +int kiro_client_connect (KiroClient *, char *, char*); int kiro_client_sync (KiroClient *); diff --git a/kiro-rdma.h b/kiro-rdma.h index ff39136..25040c8 100644 --- a/kiro-rdma.h +++ b/kiro-rdma.h @@ -77,7 +77,7 @@ struct kiro_rdma_mem { }; -static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr *mr, void *mem, size_t mem_size, int access) +static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr **mr, void *mem, size_t mem_size, int access) { if(mem_size == 0) @@ -97,8 +97,8 @@ static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr *mr, void return -1; } - mr = ibv_reg_mr(pd, mem_handle, mem_size, access); - if(!mr) + *mr = ibv_reg_mr(pd, mem_handle, mem_size, access); + if(!(*mr)) { // Memory Registration failed printf("Failed to register memory region!\n"); @@ -125,12 +125,16 @@ static struct kiro_rdma_mem* kiro_create_rdma_memory (struct ibv_pd *pd, size_t return NULL; } - if(kiro_register_rdma_memory(pd, krm->mr, krm->mem, mem_size, access)) + if(kiro_register_rdma_memory(pd, &(krm->mr), krm->mem, mem_size, access)) { free(krm); return NULL; } + if(!krm->mem) + krm->mem = krm->mr->addr; + + return krm; } diff --git a/kiro-server.c b/kiro-server.c index 145922a..119f0c1 100644 --- a/kiro-server.c +++ b/kiro-server.c @@ -99,6 +99,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port) printf("Failed to bind to address %s:%s\n",address, port); return -1; } + printf("Bound to address %s:%s\n",address, port); struct ibv_qp_init_attr qp_attr; memset(&qp_attr, 0, sizeof(qp_attr)); @@ -114,6 +115,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port) printf("Endpoint creation failed.\n"); return -1; } + printf("Endpoint created.\n"); if(rdma_listen(priv->base, 0)) { @@ -121,6 +123,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port) rdma_destroy_ep(priv->base); return -1; } + printf("Enpoint listening.\n"); priv->client = (struct kiro_connection *)calloc(1, sizeof(struct kiro_connection)); if(!priv->client) @@ -131,12 +134,14 @@ int kiro_server_start (KiroServer *self, char *address, char *port) } priv->client->identifier = 0; //First Client + printf("Waiting for connection request.\n"); if(rdma_get_request(priv->base, &(priv->client->id))) { printf("Failure waiting for clienet connection.\n"); rdma_destroy_ep(priv->base); return -1; } + printf("Connection Request received.\n"); struct kiro_connection_context *ctx = (struct kiro_connection_context *)calloc(1,sizeof(struct kiro_connection_context)); if(!ctx) @@ -174,7 +179,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port) ctx->cf_mr_recv->size = ctx->cf_mr_send->size = sizeof(struct kiro_ctrl_msg); priv->client->id->context = ctx; - if(!rdma_post_recv(priv->client->id, priv->client, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr)) + if(rdma_post_recv(priv->client->id, priv->client, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr)) { printf("Posting preemtive receive for connection failed.\n"); kiro_destroy_connection_context(ctx); diff --git a/test-client.c b/test-client.c index 1df974c..67dacf7 100644 --- a/test-client.c +++ b/test-client.c @@ -7,6 +7,6 @@ int main(void) { KiroClient *client = g_object_new(KIRO_TYPE_CLIENT, NULL); - kiro_client_connect(client, "127.0.0.1", "60010", 5000); + kiro_client_connect(client, "192.168.11.61", "60010", 5000); return 0; } \ No newline at end of file diff --git a/test-server.c b/test-server.c index cac62e6..2b08f15 100644 --- a/test-server.c +++ b/test-server.c @@ -7,6 +7,6 @@ int main(void) { KiroServer *server = g_object_new(KIRO_TYPE_SERVER, NULL); - kiro_server_start(server, "127.0.0.1", "60010"); + kiro_server_start(server, "192.168.11.61", "60010"); return 0; } \ No newline at end of file -- cgit v1.2.3