[DAPHNE-#61] Basic relational query processing based on columnar operators

UserConfig.json

@@ -6,13 +6,15 @@
     "matmul_unroll_factor": 1,
     "matmul_unroll_jam_factor": 4,
     "matmul_num_vec_registers": 16,
+    "use_columnar": false,
     "use_cuda": false,
     "use_vectorized_exec": false,
     "use_obj_ref_mgnt": true,
     "cuda_fuse_any": false,
     "use_mlir_codegen": false,
     "vectorized_single_queue": false,
     "debug_llvm": false,
+    "explain_columnar": false,
     "explain_kernels": false,
     "explain_llvm": false,
     "explain_parsing": false,

src/api/cli/DaphneUserConfig.h

@@ -36,6 +36,8 @@ class DaphneLogger;
  */
 struct DaphneUserConfig {
     // Remember to update UserConfig.json accordingly!
+
+    bool use_columnar = false;
     bool use_cuda = false;
     bool use_vectorized_exec = false;
     bool use_distributed = false;
@@ -62,6 +64,7 @@ struct DaphneUserConfig {
     bool enable_profiling = false;
 
     bool debug_llvm = false;
+    bool explain_columnar = false;
     bool explain_kernels = false;
     bool explain_llvm = false;
     bool explain_parsing = false;

src/api/internal/daphne_internal.cpp

@@ -230,6 +230,9 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
                                            "(e.g., dense/sparse)"));
     static alias selectMatrixReprAlias( // to still support the longer old form
         "select-matrix-representations", aliasopt(selectMatrixRepr), desc("Alias for --select-matrix-repr"));
+    static opt<bool> useColumnar("columnar", cat(daphneOptions),
+                                 desc("Use columnar operations instead of frame/matrix operations for relational query "
+                                      "processing where possible"));
     static opt<bool> cuda("cuda", cat(daphneOptions), desc("Use CUDA"));
     static opt<bool> fpgaopencl("fpgaopencl", cat(daphneOptions), desc("Use FPGAOPENCL"));
     static opt<string> libDir("libdir", cat(daphneOptions),
@@ -276,6 +279,7 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
                                       "(path to a kernel catalog JSON file)."));
 
     enum ExplainArgs {
+        columnar,
         kernels,
         llvm,
         parsing,
@@ -301,6 +305,7 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
             clEnumVal(parsing, "Show DaphneIR after parsing"),
             clEnumVal(parsing_simplified, "Show DaphneIR after parsing and some simplifications"),
             clEnumVal(sql, "Show DaphneIR after SQL parsing"),
+            clEnumVal(columnar, "Show DaphneIR after lowering to columnar operations"),
             clEnumVal(property_inference, "Show DaphneIR after property inference"),
             clEnumVal(select_matrix_repr, "Show DaphneIR after selecting "
                                           "physical matrix representations"),
@@ -420,6 +425,7 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
     user_config.debugMultiThreading = debugMultiThreading;
     user_config.prePartitionRows = prePartitionRows;
     user_config.distributedBackEndSetup = distributedBackEndSetup;
+    user_config.use_columnar = useColumnar;
     if (user_config.use_distributed) {
         if (user_config.distributedBackEndSetup != ALLOCATION_TYPE::DIST_MPI &&
             user_config.distributedBackEndSetup != ALLOCATION_TYPE::DIST_GRPC_SYNC &&
@@ -451,6 +457,9 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
 
     for (auto explain : explainArgList) {
         switch (explain) {
+        case columnar:
+            user_config.explain_columnar = true;
+            break;
         case kernels:
             user_config.explain_kernels = true;
             break;

src/compiler/execution/DaphneIrExecutor.cpp

@@ -117,11 +117,26 @@ bool DaphneIrExecutor::runPasses(mlir::ModuleOp module) {
     pm.addNestedPass<mlir::func::FuncOp>(mlir::daphne::createInferencePass());
     pm.addPass(mlir::createCanonicalizerPass());
 
+    if (userConfig_.use_columnar) {
+        // Rewrite certain matrix/frame ops from linear/relational algebra to columnar ops from column algebra.
+        pm.addPass(mlir::daphne::createRewriteToColumnarOpsPass());
+        // Infer the result types of the newly created columnar ops.
+        pm.addNestedPass<mlir::func::FuncOp>(mlir::daphne::createInferencePass());
+        // Simplify the IR.
+        pm.addPass(mlir::createCanonicalizerPass());
+        // Remove unused ops after simplifications.
+        // TODO The CSE pass seems to eliminate only "one row" of dead code at a time, so we need it as many times as
+        // the longest chain of ops we reduce; how to apply CSE until a fixpoint?
+        for (size_t i = 0; i < 5; i++)
+            pm.addPass(mlir::createCSEPass());
+    }
+    if (userConfig_.explain_columnar)
+        pm.addPass(mlir::daphne::createPrintIRPass("IR after lowering to columnar ops:"));
+
     if (selectMatrixRepresentations_) {
         pm.addNestedPass<mlir::func::FuncOp>(mlir::daphne::createSelectMatrixRepresentationsPass(userConfig_));
         pm.addNestedPass<mlir::func::FuncOp>(mlir::createCanonicalizerPass());
     }
-
     if (userConfig_.explain_select_matrix_repr)
         pm.addPass(mlir::daphne::createPrintIRPass("IR after selecting matrix representations:"));
 

src/compiler/inference/InferencePass.cpp

@@ -134,6 +134,8 @@ class InferencePass : public PassWrapper<InferencePass, OperationPass<func::Func
                 t = mt.withSameElementType();
             else if (auto ft = t.dyn_cast<daphne::FrameType>())
                 t = ft.withSameColumnTypes();
+            else if (auto ct = t.dyn_cast<daphne::ColumnType>())
+                t = ct.withSameValueType();
             op->getResult(i).setType(t);
         }
         return WalkResult::advance();
@@ -482,10 +484,14 @@ class InferencePass : public PassWrapper<InferencePass, OperationPass<func::Func
                 return true;
             if (auto mt = resType.dyn_cast<daphne::MatrixType>())
                 return llvm::isa<daphne::UnknownType>(mt.getElementType());
-            if (auto ft = resType.dyn_cast<daphne::FrameType>())
+            if (auto ft = resType.dyn_cast<daphne::FrameType>()) {
                 for (Type ct : ft.getColumnTypes())
                     if (llvm::isa<daphne::UnknownType>(ct))
                         return true;
+                return false;
+            }
+            if (auto ct = resType.dyn_cast<daphne::ColumnType>())
+                return ct.getValueType().isa<daphne::UnknownType>();
             return false;
         });
     }

src/compiler/inference/TypeInferenceUtils.cpp

@@ -100,6 +100,8 @@ DataTypeCode getDataTypeCode(mlir::Type t) {
         return DataTypeCode::FRAME;
     if (llvm::isa<mlir::daphne::MatrixType>(t))
         return DataTypeCode::MATRIX;
+    if (llvm::isa<mlir::daphne::ColumnType>(t))
+        return DataTypeCode::COLUMN;
     if (auto lt = t.dyn_cast<mlir::daphne::ListType>())
         return getDataTypeCode(lt.getElementType());
     if (CompilerUtils::isScaType(t))

src/compiler/inference/TypeInferenceUtils.h

@@ -46,6 +46,7 @@ enum class DataTypeCode : uint8_t {
     // The greater the number, the more general the type.
     SCALAR, // least general
     MATRIX,
+    COLUMN,
     FRAME,
     UNKNOWN // most general
 
@@ -136,6 +137,8 @@ template <class O> mlir::Type inferTypeByTraits(O *op) {
         resDtc = DataTypeCode::SCALAR;
     else if (op->template hasTrait<DataTypeMat>())
         resDtc = DataTypeCode::MATRIX;
+    else if (op->template hasTrait<DataTypeCol>())
+        resDtc = DataTypeCode::COLUMN;
     else if (op->template hasTrait<DataTypeFrm>())
         resDtc = DataTypeCode::FRAME;
 
@@ -251,8 +254,9 @@ template <class O> mlir::Type inferTypeByTraits(O *op) {
                             resVts.push_back(argVts[i][0]);
                     break;
                 }
+                case DataTypeCode::COLUMN: // fall-through intended
                 case DataTypeCode::SCALAR:
-                    // Append the value type of this input scalar to
+                    // Append the value type of this input scalar/column to
                     // the result column types.
                     resVts.push_back(argVts[i][0]);
                     break;
@@ -268,6 +272,7 @@ template <class O> mlir::Type inferTypeByTraits(O *op) {
             }
             break;
         case DataTypeCode::MATRIX: // fall-through intended
+        case DataTypeCode::COLUMN: // fall-through intended
         case DataTypeCode::SCALAR:
             resVts = {mostGeneralVt(argVts, numArgsConsider)};
             break;
@@ -286,7 +291,7 @@ template <class O> mlir::Type inferTypeByTraits(O *op) {
     // Create the result type
     // --------------------------------------------------------------------
 
-    // It is important to recreate matrix and frame types (not reuse those from
+    // It is important to recreate matrix, frame, and column types (not reuse those from
     // the inputs) to get rid of any additional properties (shape, etc.).
     switch (resDtc) {
     case DataTypeCode::UNKNOWN:
@@ -301,6 +306,9 @@ template <class O> mlir::Type inferTypeByTraits(O *op) {
     case DataTypeCode::FRAME:
         resTy = daphne::FrameType::get(ctx, resVts);
         break;
+    case DataTypeCode::COLUMN:
+        resTy = daphne::ColumnType::get(ctx, mostGeneralVt(resVts));
+        break;
     }
 
     if (resIsList)

src/compiler/lowering/CMakeLists.txt

@@ -24,6 +24,7 @@ add_mlir_dialect_library(MLIRDaphneTransforms
     LowerToLLVMPass.cpp
     PhyOperatorSelectionPass.cpp
     RewriteToCallKernelOpPass.cpp
+    RewriteToColumnarOpsPass.cpp
     SpecializeGenericFunctionsPass.cpp
     VectorizeComputationsPass.cpp
     DaphneOptPass.cpp

src/compiler/lowering/LowerToLLVMPass.cpp

@@ -913,6 +913,8 @@ void DaphneLowerToLLVMPass::runOnOperation() {
         [&](daphne::FrameType t) { return LLVM::LLVMPointerType::get(IntegerType::get(t.getContext(), 1)); });
     typeConverter.addConversion(
         [&](daphne::ListType t) { return LLVM::LLVMPointerType::get(IntegerType::get(t.getContext(), 1)); });
+    typeConverter.addConversion(
+        [&](daphne::ColumnType t) { return LLVM::LLVMPointerType::get(IntegerType::get(t.getContext(), 1)); });
     typeConverter.addConversion(
         [&](daphne::StringType t) { return LLVM::LLVMPointerType::get(IntegerType::get(t.getContext(), 8)); });
     typeConverter.addConversion([&](daphne::VariadicPackType t) {

src/compiler/lowering/RewriteToCallKernelOpPass.cpp

@@ -127,12 +127,15 @@ class KernelReplacement : public RewritePattern {
 
     mlir::Type adaptType(mlir::Type t, bool generalizeToStructure) const {
         MLIRContext *mctx = t.getContext();
-        if (generalizeToStructure && t.isa<mlir::daphne::MatrixType, mlir::daphne::FrameType, mlir::daphne::ListType>())
+        if (generalizeToStructure && t.isa<mlir::daphne::MatrixType, mlir::daphne::FrameType, mlir::daphne::ColumnType,
+                                           mlir::daphne::ListType>())
             return mlir::daphne::StructureType::get(mctx);
         if (auto mt = t.dyn_cast<mlir::daphne::MatrixType>())
             return mt.withSameElementTypeAndRepr();
         if (t.isa<mlir::daphne::FrameType>())
             return mlir::daphne::FrameType::get(mctx, {mlir::daphne::UnknownType::get(mctx)});
+        if (auto ct = t.dyn_cast<mlir::daphne::ColumnType>())
+            return ct.withSameValueType();
         if (auto lt = t.dyn_cast<mlir::daphne::ListType>())
             return mlir::daphne::ListType::get(mctx, adaptType(lt.getElementType(), generalizeToStructure));
         if (auto mrt = t.dyn_cast<mlir::MemRefType>()) {