Support forceful clean-up of kernel memory

6 files changed, 126 insertions, 36 deletions

diff --git a/NOTES b/NOTES
index 3f1eadb..47b2d51 100644
--- a/NOTES
+++ b/NOTES
@@ -32,7 +32,8 @@ DMA Access Synchronization
     
     KMEM_FLAG_EXCLUSIVE - prevents multiple processes mmaping the buffer 
     simultaneously. This is used to prevent multiple processes use the same
-    DMA engine at the same time.
+    DMA engine at the same time. When passed to kmem_free, allows to clean
+    buffers with lost clients even for shared buffers.
     
     KMEM_FLAG_REUSE - requires reuse of existing buffer. If reusable buffer is 
     found (non-reusable buffers, i.e. allocated without KMEM_FLAG_REUSE are
diff --git a/cli.c b/cli.c
index 5deee17..de796dc 100644
--- a/cli.c
+++ b/cli.c
@@ -92,6 +92,7 @@ typedef enum {
     OPT_ITERATIONS,
     OPT_LIST_KMEM,
     OPT_FREE_KMEM,
+    OPT_FORCE,
     OPT_HELP = 'h',
 } OPTIONS;
 
@@ -116,8 +117,9 @@ static struct option long_options[] = {
     {"stop-dma",		optional_argument, 0, OPT_STOP_DMA },
     {"wait-irq",		optional_argument, 0, OPT_WAIT_IRQ },
     {"list-kernel-memory",	no_argument, 0, OPT_LIST_KMEM },
-    {"free-kernel-memory",	optional_argument, 0, OPT_FREE_KMEM },
+    {"free-kernel-memory",	required_argument, 0, OPT_FREE_KMEM },
     {"quiete",			no_argument, 0, OPT_QUIETE },
+    {"force",			no_argument, 0, OPT_FORCE },
     {"help",			no_argument, 0, OPT_HELP },
     { 0, 0, 0, 0 }
 };
@@ -157,7 +159,9 @@ void Usage(int argc, char *argv[], const char *format, ...) {
 "\n"
 "  Kernel Modes:\n"
 "   --list-kernel-memory 	- List kernel buffers\n"
-"   --free-kernel-memory [use]	- Cleans lost kernel space buffers (DANGEROUS)\n"
+"   --free-kernel-memory <use>	- Cleans lost kernel space buffers (DANGEROUS)\n"
+"	dma			- Remove all buffers allocated by DMA subsystem\n"
+"	#number			- Remove all buffers with the specified use id\n"
 "\n"
 "  Addressing:\n"
 "   -d <device>			- FPGA device (/dev/fpga0)\n"
@@ -1087,6 +1091,38 @@ int ListKMEM(pcilib_t *handle, const char *device) {
     return 0;
 }
 
+int FreeKMEM(pcilib_t *handle, const char *device, const char *use, int force) {
+    int err;
+    int i;
+    
+    unsigned long useid;
+    
+    pcilib_kmem_flags_t flags = PCILIB_KMEM_FLAG_HARDWARE|PCILIB_KMEM_FLAG_PERSISTENT|PCILIB_KMEM_FLAG_EXCLUSIVE; 
+    if (force) flags |= PCILIB_KMEM_FLAG_FORCE; // this will ignore mmap locks as well.
+
+    if (!strcasecmp(use, "dma")) {
+	for (i = 0; i < PCILIB_MAX_DMA_ENGINES; i++) {
+	    err = pcilib_clean_kernel_memory(handle, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_RING, i), flags);
+	    if (err) Error("Error cleaning DMA%i C2S Ring buffer", i);
+	    err = pcilib_clean_kernel_memory(handle, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_RING, 0x80|i), flags);
+	    if (err) Error("Error cleaning DMA%i S2C Ring buffer", i);
+	    err = pcilib_clean_kernel_memory(handle, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, i), flags);
+	    if (err) Error("Error cleaning DMA%i C2S Page buffers", i);
+	    err = pcilib_clean_kernel_memory(handle, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, 0x80|i), flags);
+	    if (err) Error("Error cleaning DMA%i S2C Page buffers", i);
+	}
+	
+	return 0;
+    }
+    
+    if ((!isxnumber(use))||(sscanf(use, "%lx", &useid) != 1)) Error("Invalid use (%s) is specified", use);
+    
+    err = pcilib_clean_kernel_memory(handle, useid, flags);
+    if (err) Error("Error cleaning kernel buffers for use (0x%lx)", useid);
+    
+    return 0;
+}
+
 int WaitIRQ(pcilib_t *handle, pcilib_model_description_t *model_info, pcilib_irq_source_t irq_source, pcilib_timeout_t timeout) {
     int err;
     size_t count;
@@ -1113,6 +1149,7 @@ int main(int argc, char **argv) {
 
     int details = 0;
     int quiete = 0;
+    int force = 0;
     
     pcilib_model_t model = PCILIB_MODEL_DETECT;
     pcilib_model_description_t *model_info;
@@ -1311,6 +1348,9 @@ int main(int argc, char **argv) {
 	    case OPT_QUIETE:
 		quiete = 1;
 	    break;
+	    case OPT_FORCE:
+		force = 1;
+	    break;
 	    default:
 		Usage(argc, argv, "Unknown option (%s)", argv[optind]);
 	}
@@ -1494,6 +1534,9 @@ int main(int argc, char **argv) {
      case MODE_LIST_KMEM:
         ListKMEM(handle, fpga_device);
      break;
+     case MODE_FREE_KMEM:
+        FreeKMEM(handle, fpga_device, use, force);
+     break;
     }
 
     pcilib_close(handle);
diff --git a/driver/kmem.c b/driver/kmem.c
index 90ae5ba..31fc685 100644
--- a/driver/kmem.c
+++ b/driver/kmem.c
@@ -152,56 +152,90 @@ kmem_alloc_entry_fail:
 		return -ENOMEM;
 }
 
-/**
- *
- * Called via sysfs, frees kernel memory and the corresponding management structure
- *
- */
-int pcidriver_kmem_free( pcidriver_privdata_t *privdata, kmem_handle_t *kmem_handle )
-{
-	pcidriver_kmem_entry_t *kmem_entry;
-
-	/* Find the associated kmem_entry for this buffer */
-	if ((kmem_entry = pcidriver_kmem_find_entry(privdata, kmem_handle)) == NULL)
-		return -EINVAL;					/* kmem_handle is not valid */
-
-//	if (kmem_entry->id == 0)
-//	mod_info("1: %i %x %lx %lx\n", kmem_entry->id, kmem_handle->flags, kmem_entry->refs, kmem_entry->mode);
-
-	if (kmem_entry->mode&KMEM_MODE_COUNT)
+static int pcidriver_kmem_free_check(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_handle, pcidriver_kmem_entry_t *kmem_entry) {
+	if ((kmem_handle->flags & KMEM_FLAG_FORCE) == 0) {
+	    if (kmem_entry->mode&KMEM_MODE_COUNT)
 		kmem_entry->mode -= 1;
 	
-	if (kmem_handle->flags&KMEM_FLAG_HW)
+	    if (kmem_handle->flags&KMEM_FLAG_HW)
 		kmem_entry->refs &= ~KMEM_REF_HW;
 	
-	if (kmem_handle->flags&KMEM_FLAG_PERSISTENT)
+	    if (kmem_handle->flags&KMEM_FLAG_PERSISTENT)
 		kmem_entry->mode &= ~KMEM_MODE_PERSISTENT;
-
-//	if (kmem_entry->id == 0)
-//	mod_info("2: %i %x %lx %lx\n", kmem_entry->id, kmem_handle->flags, kmem_entry->refs, kmem_entry->mode);
 	
-	if (kmem_handle->flags&KMEM_FLAG_REUSE) 
+	    if (kmem_handle->flags&KMEM_FLAG_REUSE) 
 		return 0;
 
-	if (kmem_entry->refs) {
+	    if (kmem_entry->refs) {
 		mod_info("can't free referenced kmem_entry\n");
 		kmem_entry->mode += 1;
 		return -EBUSY;
-	}
+	    }
 	
-	if (kmem_entry->mode & KMEM_MODE_PERSISTENT) {
+	    if (kmem_entry->mode & KMEM_MODE_PERSISTENT) {
 		mod_info("can't free persistent kmem_entry\n");
 		return -EBUSY;
-	}
+	    }
 
-	if (((kmem_entry->mode&KMEM_MODE_EXCLUSIVE)==0)&&(kmem_entry->mode&KMEM_MODE_COUNT)) 
+	    if (((kmem_entry->mode&KMEM_MODE_EXCLUSIVE)==0)&&(kmem_entry->mode&KMEM_MODE_COUNT)&&((kmem_handle->flags&KMEM_FLAG_EXCLUSIVE)==0)) 
 		return 0;
+	}
 	
+	return 1;
+}
+
+static int pcidriver_kmem_free_use(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_handle)
+{
+	int err;
+	int failed = 0;
+	struct list_head *ptr, *next;
+	pcidriver_kmem_entry_t *kmem_entry;
+
+	/* iterate safely over the entries and delete them */
+	list_for_each_safe(ptr, next, &(privdata->kmem_list)) {
+		kmem_entry = list_entry(ptr, pcidriver_kmem_entry_t, list);
+		if (kmem_entry->use == kmem_handle->use) {
+		    err = pcidriver_kmem_free_check(privdata, kmem_handle, kmem_entry);
+		    if (err > 0)
+			pcidriver_kmem_free_entry(privdata, kmem_entry); 		/* spin lock inside! */
+		    else
+			failed = 1;
+		}
+	}
+	
+	if (failed) {
+		mod_info("Some kmem_entries for use %lx are still referenced\n", kmem_handle->use);
+		return -EBUSY;
+	}	
+
+	return 0;
+}
+
+/**
+ *
+ * Called via sysfs, frees kernel memory and the corresponding management structure
+ *
+ */
+int pcidriver_kmem_free( pcidriver_privdata_t *privdata, kmem_handle_t *kmem_handle )
+{
+	int err;
+	pcidriver_kmem_entry_t *kmem_entry;
+
+	if (kmem_handle->flags&KMEM_FLAG_MASS) {
+	    kmem_handle->flags &= ~KMEM_FLAG_MASS;
+	    return pcidriver_kmem_free_use(privdata, kmem_handle);
+	}
+	
+	/* Find the associated kmem_entry for this buffer */
+	if ((kmem_entry = pcidriver_kmem_find_entry(privdata, kmem_handle)) == NULL)
+		return -EINVAL;					/* kmem_handle is not valid */
+
+	err = pcidriver_kmem_free_check(privdata, kmem_handle, kmem_entry);
 	
-//	if (kmem_entry->id == 0)
-//	mod_info("cleaned %i\n", kmem_entry->id);
+	if (err > 0)
+		return pcidriver_kmem_free_entry(privdata, kmem_entry);
 	
-	return pcidriver_kmem_free_entry(privdata, kmem_entry);
+	return err;
 }
 
 /**
diff --git a/driver/pciDriver.h b/driver/pciDriver.h
index 9aa580f..fc92b11 100644
--- a/driver/pciDriver.h
+++ b/driver/pciDriver.h
@@ -104,13 +104,14 @@
 #define KMEM_FLAG_EXCLUSIVE 2	/**< Allow only a single application accessing a specified use & item */
 #define KMEM_FLAG_PERSISTENT 4	/**< Sets persistent mode */
 #define KMEM_FLAG_HW 8		/**< The buffer may be accessed by hardware, the hardware access will not occur any more if passed to _free function */
+#define KMEM_FLAG_FORCE 16	/**< Force memory cleanup even if references are present */
+#define KMEM_FLAG_MASS 32	/**< Apply to all buffers of selected use */
 
 #define KMEM_FLAG_REUSED 1		/**< Indicates if buffer with specified use & item was already allocated and reused */
 #define KMEM_FLAG_REUSED_PERSISTENT 4	/**< Indicates that reused buffer was persistent before the call */
 #define KMEM_FLAG_REUSED_HW 8		/**< Indicates that reused buffer had a HW reference before the call */
 
 
-
 /* Types */
 typedef struct {
 	unsigned long type;
diff --git a/kmem.c b/kmem.c
index 0ad9b39..cc84e8c 100644
--- a/kmem.c
+++ b/kmem.c
@@ -17,6 +17,15 @@
 #include "kmem.h"
 #include "error.h"
 
+int pcilib_clean_kernel_memory(pcilib_t *ctx, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
+    kmem_handle_t kh = {0};
+    kh.use = use;
+    kh.flags = flags|KMEM_FLAG_MASS;
+
+    return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
+}
+
+
 static int pcilib_free_kernel_buffer(pcilib_t *ctx, pcilib_kmem_list_t *kbuf, size_t i, pcilib_kmem_flags_t flags) {
     kmem_handle_t kh = {0};
 
diff --git a/kmem.h b/kmem.h
index de2ebdd..0f76025 100644
--- a/kmem.h
+++ b/kmem.h
@@ -9,7 +9,7 @@ typedef enum {
     PCILIB_KMEM_FLAG_EXCLUSIVE = KMEM_FLAG_EXCLUSIVE,
     PCILIB_KMEM_FLAG_PERSISTENT = KMEM_FLAG_PERSISTENT,
     PCILIB_KMEM_FLAG_HARDWARE = KMEM_FLAG_HW,
-//    PCILIB_KMEM_FLAG_FORCE = 2		/**< Force buffer 
+    PCILIB_KMEM_FLAG_FORCE = KMEM_FLAG_FORCE
 } pcilib_kmem_flags_t;
 
 
@@ -69,4 +69,6 @@ uintptr_t pcilib_kmem_get_block_pa(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_
 size_t pcilib_kmem_get_block_size(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block);
 pcilib_kmem_reuse_state_t pcilib_kmem_is_reused(pcilib_t *ctx, pcilib_kmem_handle_t *k);
 
+int pcilib_clean_kernel_memory(pcilib_t *ctx, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags);
+
 #endif /* _PCILIB_KMEM_H */