add plugin functionality

This enables the user to run custom code during database generation. The
user can populate the database with new tables, modify entries, etc.

Author: bertschinger

include/bf.h

@@ -70,6 +70,7 @@ OF SUCH DAMAGE.
 
 #include "SinglyLinkedList.h"
 #include "compress.h"
+#include "plugin.h"
 #include "trie.h"
 #include "xattrs.h"
 
@@ -274,6 +275,12 @@ struct input {
 
    /* prefix of swap files */
    refstr_t swap_prefix;
+
+   /*
+    * Holds pointers to plugin functions for running custom user code to manipulate the databases.
+    * These will be NULL if no plugin was specified.
+    */
+   const struct plugin_operations *plugin_ops;
 };
 
 struct input *input_init(struct input *in);

include/plugin.h

@@ -0,0 +1,97 @@
+/*
+This file is part of GUFI, which is part of MarFS, which is released
+under the BSD license.
+
+
+Copyright (c) 2017, Los Alamos National Security (LANS), LLC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+From Los Alamos National Security, LLC:
+LA-CC-15-039
+
+Copyright (c) 2017, Los Alamos National Security, LLC All rights reserved.
+Copyright 2017. Los Alamos National Security, LLC. This software was produced
+under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
+Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
+the U.S. Department of Energy. The U.S. Government has rights to use,
+reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR LOS
+ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
+ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is
+modified to produce derivative works, such modified software should be
+clearly marked, so as not to confuse it with the version available from
+LANL.
+
+THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.
+*/
+
+#ifndef PLUGIN_H
+#define PLUGIN_H
+
+#include <sqlite3.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Operations for a user-defined plugin library, allowing the user to make custom
+ * modifications to the databases as GUFI runs.
+ */
+struct plugin_operations {
+    /*
+     * Give the user an opportunity to initialize any state. The returned pointer
+     * is passed to all subsequent operations as `user_data`.
+     */
+    void *(*db_init)(sqlite3 *db);
+    /* Process a directory. */
+    void (*process_dir)(char *path, sqlite3 *db, void *user_data);
+    /* Process a file */
+    void (*process_file)(char *path, sqlite3 *db, void *user_data);
+    /*
+     * Clean up any state, and write out any final data to the database, for example
+     * a summary containing information from each of the files seen.
+     */
+    void (*db_exit)(sqlite3 *db, void *user_data);
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/bf.c

@@ -62,6 +62,7 @@ OF SUCH DAMAGE.
 
 
 
+#include <dlfcn.h>
 #include <pwd.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -72,6 +73,7 @@ OF SUCH DAMAGE.
 #include "bf.h"
 #include "dbutils.h"
 #include "external.h"
+#include "plugin.h"
 #include "utils.h"
 
 #define XSTR(x) #x
@@ -90,6 +92,13 @@ const char fielddelim = '\x1E';     /* ASCII Record Separator */
 
 static const char DEFAULT_SWAP_PREFIX[] = "";
 
+static const struct plugin_operations null_plugin_ops = {
+    .db_init = NULL,
+    .process_file = NULL,
+    .process_dir = NULL,
+    .db_exit = NULL,
+};
+
 struct input *input_init(struct input *in) {
     if (in) {
         memset(in, 0, sizeof(*in));
@@ -118,6 +127,8 @@ struct input *input_init(struct input *in) {
 
         in->swap_prefix.data = DEFAULT_SWAP_PREFIX;
         in->swap_prefix.len  = 0;
+
+        in->plugin_ops = &null_plugin_ops;
     }
 
     return in;
@@ -137,6 +148,30 @@ void input_fini(struct input *in) {
     }
 }
 
+/*
+ * If a plugin library path is given in in->plugin_name,
+ * then load that library and add its plugin functions to in->plugin_ops.
+ *
+ * Returns 0 on success or 1 on failure.
+ */
+int load_plugin_library(struct input *in, char *plugin_name) {
+    void *lib = dlopen(plugin_name, RTLD_NOW);
+    if (!lib) {
+        fprintf(stderr, "Could not open plugin library: %s\n", dlerror());
+        return 1;
+    }
+
+    struct plugin_operations *user_plugin_ops = (struct plugin_operations *) dlsym(lib, "exported_operations");
+    if (!user_plugin_ops) {
+        fprintf(stderr, "Could not find exported operations in the plugin library: %s\n", dlerror());
+        return 1;
+    }
+
+    in->plugin_ops = user_plugin_ops;
+
+    return 0;
+}
+
 void print_help(const char* prog_name,
                 const char* getopt_str,
                 const char* positional_args_help_str) {
@@ -198,6 +233,7 @@ void print_help(const char* prog_name,
                        "     <view>              External database file basename, per-attach table name, template + table name, and the resultant view"); break;
       case 's': printf("  -s <path>              File name prefix for swap files"); break;
       case 'p': printf("  -p <path>              Source path prefix for %%s in SQL"); break;
+      case 'U': printf("  -U <library_name>      plugin library for modifying db entries"); break;
       default: printf("print_help(): unrecognized option '%c'", (char)ch);
       }
       printf("\n");
@@ -518,6 +554,12 @@ int parse_cmd_line(int         argc,
           INSTALL_STR(&in->sql_format.source_prefix, optarg);
           break;
 
+      case 'U':
+          if (load_plugin_library(in, optarg)) {
+              retval = -1;
+          }
+          break;
+
       case '?':
          // getopt returns '?' when there is a problem.  In this case it
          // also prints, e.g. "getopt_test: illegal option -- z"

test/regression/gufi_dir2index.expected

@@ -13,6 +13,7 @@ options:
   -M <bytes>             target memory footprint
   -s <path>              File name prefix for swap files
   -C <count>             Number of subdirectories allowed to be enqueued for parallel processing. Any remainders will be processed in-situ
+  -U <library_name>      plugin library for modifying db entries
   -e                     compress work items
   -q                     check that external databases are valid before tracking during indexing