Merge pull request #2184 from rapidsai/branch-22.04

[RELEASE] cugraph v22.04

Author: raydouglass

.github/ops-bot.yaml

@@ -0,0 +1,8 @@
+# This file controls which features from the `ops-bot` repository below are enabled.
+# - https://github.com/rapidsai/ops-bot
+
+auto_merger: true
+branch_checker: true
+label_checker: true
+release_drafter: true
+external_contributors: false

.gitignore

@@ -79,11 +79,6 @@ datasets/*
 ## Doxygen
 cpp/doxygen/html
 
-# Raft symlink
-python/cugraph/cugraph/raft
-python/pylibcugraph/pylibcugraph/raft
-python/_external_repositories/
-
 # created by Dask tests
 python/dask-worker-space
 python/cugraph/dask-worker-space

CHANGELOG.md

@@ -39,6 +39,23 @@ There are 3 ways to get cuGraph :
 <br/><br/>
 
 ---
+# cuGraph News
+
+### Scaling to 1 Trillion Edges
+cuGraph was recently tested on the Selene supercomputer using 2,048 GPUs and processing a graph with `1.1 Trillion edges`. 
+
+<div align="left"><img src="img/Scaling.png" width="500px" style="background-color: white;"/>&nbsp;</br>cuGraph Scaling</div>
+</br></br>
+
+### cuGraph Software Stack
+cuGraph has a new multi-layer software stack that allows users and system integrators to access cuGraph at different layers.  
+
+<div align="left"><img src="img/cugraph-stack.png" width="500px" style="background-color: white;"/>&nbsp;</br>cuGraph Software Stack</div>
+</br></br>
+
+
+
+
 # Currently Supported Features
 As of Release 21.08 - including 21.08 nightly
 
@@ -50,24 +67,24 @@ _Italic_ algorithms are planned for future releases.
 | ------------ | -------------------------------------- | ------------ | ------------------- |
 | Centrality   |                                        |              |                     |
 |              | Katz                                   | Multi-GPU    |                     |
-|              | Betweenness Centrality                 | Single-GPU   |                     |
+|              | Betweenness Centrality                 | Single-GPU   | MG planned for 22.08                    |
 |              | Edge Betweenness Centrality            | Single-GPU   |                     |
+|              | _Eigenvector Centrality_               |              | _MG planned for 22.06_ |
 | Community    |                                        |              |                     |
-|              | EgoNet                                 | Single-GPU   |                     |
 |              | Leiden                                 | Single-GPU   |                     |
 |              | Louvain                                | Multi-GPU    |  [C++ README](cpp/src/community/README.md#Louvain) |
 |              | Ensemble Clustering for Graphs         | Single-GPU   |                     |
 |              | Spectral-Clustering - Balanced Cut     | Single-GPU   |                     |
 |              | Spectral-Clustering - Modularity       | Single-GPU   |                     |
 |              | Subgraph Extraction                    | Single-GPU   |                     |
-|              | Triangle Counting                      | Single-GPU   |                     |
-|              | K-Truss                                | Single-GPU   |                     |
+|              | Triangle Counting                      | Single-GPU   | MG planned for 22.06                    |
+|              | K-Truss                                | Single-GPU   | MG planned for 22.10                    |
 | Components   |                                        |              |                     |
 |              | Weakly Connected Components            | Multi-GPU    |                     |
-|              | Strongly Connected Components          | Single-GPU   |                     |
+|              | Strongly Connected Components          | Single-GPU   | MG planned for 22.06      |
 | Core         |                                        |              |                     |
-|              | K-Core                                 | Single-GPU   |                     |
-|              | Core Number                            | Single-GPU   |                     |
+|              | K-Core                                 | Single-GPU   | MG planned for 22.10                    |
+|              | Core Number                            | Single-GPU   | MG planned for 22.08                    |
 | _Flow_       |                                        |              |                     |
 |              | _MaxFlow_                              | ---          |                     |
 | _Influence_  |                                        |              |                     |
@@ -79,7 +96,7 @@ _Italic_ algorithms are planned for future releases.
 | Link Analysis|                                        |              |                     |
 |              | Pagerank                               | Multi-GPU    | [C++ README](cpp/src/centrality/README.md#Pagerank) |
 |              | Personal Pagerank                      | Multi-GPU    | [C++ README](cpp/src/centrality/README.md#Personalized-Pagerank) |
-|              | HITS                                   | Single-GPU   | Multi-GPU C code is ready, Python wrapper in 22.04                    |
+|              | HITS                                   | Multi-GPU    |                     |
 | Link Prediction |                                     |              |                     |
 |              | Jaccard Similarity                     | Single-GPU   |                     |
 |              | Weighted Jaccard Similarity            | Single-GPU   |                     |
@@ -89,10 +106,12 @@ _Italic_ algorithms are planned for future releases.
 | Sampling     |                                        |              |                     |
 |              | Random Walks (RW)                      | Single-GPU   | Biased and Uniform  |
 |              | Egonet                                 | Single-GPU   | multi-seed          |
-|              | _node2vec_                             |   ---        | C code is ready, Python wrapper coming in 22.04                    |
+|              | Node2Vec                               | Single-GPU   |                     |
+|              | Neighborhood sampling                  | Multi-GPU    |                     |
 | Traversal    |                                        |              |                     |
 |              | Breadth First Search (BFS)             | Multi-GPU    | with cutoff support <br/> [C++ README](cpp/src/traversal/README.md#BFS) |
 |              | Single Source Shortest Path (SSSP)     | Multi-GPU    | [C++ README](cpp/src/traversal/README.md#SSSP) |
+|              | _ASSP / APSP_                          |              |                     |
 | Tree         |                                        |              |                     |
 |              | Minimum Spanning Tree                  | Single-GPU   |                     |
 |              | Maximum Spanning Tree                  | Single-GPU   |                     |
@@ -164,20 +183,20 @@ Install and update cuGraph using the conda command:
 
 ```bash
 
-# CUDA 11.0
-conda install -c nvidia -c rapidsai -c numba -c conda-forge cugraph cudatoolkit=11.0
 
-# CUDA 11.2
-conda install -c nvidia -c rapidsai -c numba -c conda-forge cugraph cudatoolkit=11.2
+
+
 
 # CUDA 11.4
 conda install -c nvidia -c rapidsai -c numba -c conda-forge cugraph cudatoolkit=11.4
 
 # CUDA 11.5
 conda install -c nvidia -c rapidsai -c numba -c conda-forge cugraph cudatoolkit=11.5
+
+For CUDA > 11.5, please use the 11.5 environment
 ```
 
-Note: This conda installation only applies to Linux and Python versions 3.7/3.8.
+Note: This conda installation only applies to Linux and Python versions 3.8/3.9.
 
 
 ## Build from Source and Contributing <a name="source"></a>

README.md

@@ -75,7 +75,7 @@ conda activate cugraph_dev
 
 
 ### Build and Install Using the `build.sh` Script
-Using the `build.sh` script make compiling and installig cuGraph a breeze.  To build and install, simply do:
+Using the `build.sh` script make compiling and installing cuGraph a breeze.  To build and install, simply do:
 
 ```bash
 $ cd $CUGRAPH_HOME
@@ -144,6 +144,30 @@ This project uses cmake for building the C/C++ library. To configure cmake, run:
   ```
 The default installation locations are `$CMAKE_INSTALL_PREFIX/lib` and `$CMAKE_INSTALL_PREFIX/include/cugraph` respectively.
 
+#### Updating the RAFT branch
+
+`libcugraph` uses the [RAFT](https://github.com/rapidsai/raft) library and there are times when it might be desirable to build against a different RAFT branch, such as when working on new features that might span both RAFT and cuGraph. 
+
+For local development, the `CPM_raft_SOURCE=<path/to/raft/source>` option can be passed to the `cmake` command to enable `libcugraph` to use the local RAFT branch.
+
+To have CI test a `cugraph` pull request against a different RAFT branch, modify the bottom of the `cpp/cmake/thirdparty/get_raft.cmake` file as follows:
+
+```cmake
+# Change pinned tag and fork here to test a commit in CI
+# To use a different RAFT locally, set the CMake variable
+# RPM_raft_SOURCE=/path/to/local/raft
+find_and_configure_raft(VERSION    ${CUGRAPH_MIN_VERSION_raft}
+                        FORK       <your_git_fork>
+                        PINNED_TAG <your_git_branch_or_tag>
+
+                        # When PINNED_TAG above doesn't match cugraph,
+                        # force local raft clone in build directory
+                        # even if it's already installed.
+                        CLONE_ON_PIN     ON
+                        )
+```
+
+When the above change is pushed to a pull request, the continuous integration servers will use the specified RAFT branch to run the cuGraph tests. After the changes in the RAFT branch are merged to the release branch, remember to revert the `get_raft.cmake` file back to the original cuGraph branch.
 
 ### Building and installing the Python package
 

SOURCEBUILD.md

@@ -53,7 +53,7 @@ BUILD_DIRS="${LIBCUGRAPH_BUILD_DIR} ${LIBCUGRAPH_ETL_BUILD_DIR} ${CUGRAPH_BUILD_
 VERBOSE_FLAG=""
 CMAKE_VERBOSE_OPTION=""
 BUILD_TYPE=Release
-INSTALL_TARGET=install
+INSTALL_TARGET="--target install"
 BUILD_CPP_TESTS=ON
 BUILD_CPP_MG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
@@ -198,7 +198,7 @@ if buildAll || hasArg libcugraph; then
           -DBUILD_TESTS=${BUILD_CPP_TESTS} \
           -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \
           ${CMAKE_VERBOSE_OPTION}
-    cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} --target ${INSTALL_TARGET} ${VERBOSE_FLAG}
+    cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} ${INSTALL_TARGET} ${VERBOSE_FLAG}
 fi
 
 # Configure, build, and install libcugraph_etl
@@ -220,7 +220,7 @@ if buildAll || hasArg libcugraph_etl; then
           -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \
           ${CMAKE_VERBOSE_OPTION} \
           ${REPODIR}/cpp/libcugraph_etl
-    cmake --build "${LIBCUGRAPH_ETL_BUILD_DIR}" -j${PARALLEL_LEVEL} --target ${INSTALL_TARGET} ${VERBOSE_FLAG}
+    cmake --build "${LIBCUGRAPH_ETL_BUILD_DIR}" -j${PARALLEL_LEVEL} ${INSTALL_TARGET} ${VERBOSE_FLAG}
 fi
 
 # Build, and install pylibcugraph

build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright (c) 2018-2021, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 ##########################################
 # cuGraph Benchmark test script for CI   #
 ##########################################
@@ -48,7 +48,7 @@ export DATASETS_DIR=${WORKSPACE}/datasets
 export BENCHMARKS_DIR=${WORKSPACE}/benchmarks
 
 # ucx-py version
-export UCX_PY_VERSION='0.24.*'
+export UCX_PY_VERSION='0.25.*'
 
 ##########################################
 # Environment Setup                      #

ci/benchmark/build.sh

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,17 @@
 import argparse
 import io
 import os
-import git_helpers
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.expanduser(__file__)))
+
+# Add the scripts dir for gitutils
+sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR,
+                                              "../../cpp/scripts")))
+
+# Now import gitutils. Ignore flake8 error here since there is no other way to
+# set up imports
+import gitutils  # noqa: E402
 
 FilesToCheck = [
     re.compile(r"[.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx)$"),
@@ -28,22 +38,25 @@
     re.compile(r"[.]flake8[.]cython$"),
     re.compile(r"meta[.]yaml$")
 ]
+ExemptFiles = []
 
 # this will break starting at year 10000, which is probably OK :)
-CheckSimple = re.compile(r"Copyright \(c\) (\d{4}), NVIDIA CORPORATION")
+CheckSimple = re.compile(
+    r"Copyright *(?:\(c\))? *(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)")
 CheckDouble = re.compile(
-    r"Copyright \(c\) (\d{4})-(\d{4}), NVIDIA CORPORATION")
+    r"Copyright *(?:\(c\))? *(\d{4})-(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)"  # noqa: E501
+)
 
 
 def checkThisFile(f):
     # This check covers things like symlinks which point to files that DNE
-    if not(os.path.exists(f)):
+    if not (os.path.exists(f)):
         return False
-    if git_helpers and git_helpers.isFileEmpty(f):
-        return False
-    # Special case for versioneer.py - it uses a separate copyright.
-    if os.path.basename(f) == "versioneer.py":
+    if gitutils and gitutils.isFileEmpty(f):
         return False
+    for exempt in ExemptFiles:
+        if exempt.search(f):
+            return False
     for checker in FilesToCheck:
         if checker.search(f):
             return True
@@ -87,12 +100,22 @@ def checkCopyright(f, update_current_year):
             continue
         crFound = True
         if start > end:
-            e = [f, lineNum, "First year after second year in the copyright "
-                 "header (manual fix required)", None]
+            e = [
+                f,
+                lineNum,
+                "First year after second year in the copyright "
+                "header (manual fix required)",
+                None
+            ]
             errs.append(e)
         if thisYear < start or thisYear > end:
-            e = [f, lineNum, "Current year not included in the "
-                 "copyright header", None]
+            e = [
+                f,
+                lineNum,
+                "Current year not included in the "
+                "copyright header",
+                None
+            ]
             if thisYear < start:
                 e[-1] = replaceCurrentYear(line, thisYear, end)
             if thisYear > end:
@@ -103,8 +126,13 @@ def checkCopyright(f, update_current_year):
     fp.close()
     # copyright header itself not found
     if not crFound:
-        e = [f, 0, "Copyright header missing or formatted incorrectly "
-             "(manual fix required)", None]
+        e = [
+            f,
+            0,
+            "Copyright header missing or formatted incorrectly "
+            "(manual fix required)",
+            None
+        ]
         errs.append(e)
     # even if the year matches a copyright header, make the check pass
     if yearMatched:
@@ -125,7 +153,6 @@ def checkCopyright(f, update_current_year):
     return errs
 
 
-
 def getAllFilesUnderDir(root, pathFilter=None):
     retList = []
     for (dirpath, dirnames, filenames) in os.walk(root):
@@ -143,25 +170,47 @@ def checkCopyright_main():
     it compares between branches "$PR_TARGET_BRANCH" and "current-pr-branch"
     """
     retVal = 0
+    global ExemptFiles
 
     argparser = argparse.ArgumentParser(
-        description="Checks for a consistent copyright header")
-    argparser.add_argument("--update-current-year", dest='update_current_year',
-                           action="store_true", required=False, help="If set, "
-                           "update the current year if a header is already "
-                           "present and well formatted.")
-    argparser.add_argument("--git-modified-only", dest='git_modified_only',
-                           action="store_true", required=False, help="If set, "
-                           "only files seen as modified by git will be "
-                           "processed.")
+        "Checks for a consistent copyright header in git's modified files")
+    argparser.add_argument("--update-current-year",
+                           dest='update_current_year',
+                           action="store_true",
+                           required=False,
+                           help="If set, "
+                                "update the current year if a header "
+                                "is already present and well formatted.")
+    argparser.add_argument("--git-modified-only",
+                           dest='git_modified_only',
+                           action="store_true",
+                           required=False,
+                           help="If set, "
+                                "only files seen as modified by git will be "
+                                "processed.")
+    argparser.add_argument("--exclude",
+                           dest='exclude',
+                           action="append",
+                           required=False,
+                           default=["python/cuml/_thirdparty/"],
+                           help=("Exclude the paths specified (regexp). "
+                                 "Can be specified multiple times."))
 
     (args, dirs) = argparser.parse_known_args()
+    try:
+        ExemptFiles = ExemptFiles + [pathName for pathName in args.exclude]
+        ExemptFiles = [re.compile(file) for file in ExemptFiles]
+    except re.error as reException:
+        print("Regular expression error:")
+        print(reException)
+        return 1
+
     if args.git_modified_only:
-        files = git_helpers.modifiedFiles(pathFilter=checkThisFile)
+        files = gitutils.modifiedFiles(pathFilter=checkThisFile)
     else:
         files = []
         for d in [os.path.abspath(d) for d in dirs]:
-            if not(os.path.isdir(d)):
+            if not (os.path.isdir(d)):
                 raise ValueError(f"{d} is not a directory.")
             files += getAllFilesUnderDir(d, pathFilter=checkThisFile)
 
@@ -178,8 +227,9 @@ def checkCopyright_main():
         path_parts = os.path.abspath(__file__).split(os.sep)
         file_from_repo = os.sep.join(path_parts[path_parts.index("ci"):])
         if n_fixable > 0:
-            print("You can run {} --update-current-year to fix {} of these "
-                  "errors.\n".format(file_from_repo, n_fixable))
+            print(("You can run `python {} --git-modified-only "
+                   "--update-current-year` to fix {} of these "
+                   "errors.\n").format(file_from_repo, n_fixable))
         retVal = 1
     else:
         print("Copyright check passed")