Adding Unit test for coverage

.github/workflows/build.yaml

@@ -10,37 +10,90 @@
 name: build
 
 on: [push, pull_request, workflow_dispatch]
-# workflow_dispatch should make manually triggered ci/cd possible
-# workflow file (like this) with `workflow_dispatch` after on should exist on the **master** or default branch,
-# or there will be no ui for a manual trigger. https://github.community/t/workflow-dispatch-event-not-working/128856/2
+ # workflow_dispatch should make manually triggered ci/cd possible
+ # workflow file (like this) with `workflow_dispatch` after on should exist on the **master** or default branch,
+ # or there will be no ui for a manual trigger. https://github.community/t/workflow-dispatch-event-not-working/128856/2
+
+env:
+  REPORTS_LOCATIONS: |
+    pravega-sensor-collector/build/reports
+
 
 jobs:
   build:
     name: Build
     runs-on: ubuntu-20.04
     steps:
+      - name: Build Information
+        run: echo Building a '${{ github.event_name }}' for target '${{ github.ref }}'.
       - name: Checkout
         uses: actions/checkout@v2
-
-      - name: Set up JDK
-        uses: actions/setup-java@v2
+      - uses: actions/setup-java@v2
         with:
-          distribution: 'zulu'
-          java-version: '11' # major or semver Java version will be acceptable, see https://github.com/marketplace/actions/setup-java-jdk#basic
-
-      - name: Build
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Test build-installer.sh
         run: ./scripts/build-installer.sh
+      - name: Clean
+        if: ${{ startsWith(github.ref, 'refs/heads/') }}
+        run: ./gradlew clean ${{env.GRADLE_OPTS}}
+      - name: Compile & Checkstyle
+        run: ./gradlew jar compileTest checkstyleMain checkstyleTest --parallel ${{env.GRADLE_OPTS}}
+      - name: Spotbugs
+        run: ./gradlew spotbugsMain spotbugsTest
+      - name: Tar Reports
+        run: tar --use-compress-program zstd -cf reports-${{github.job}}.tzst `echo ${{env.REPORTS_LOCATIONS}}`
+      - name: Upload Reports
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{github.job}}-reports
+          retention-days: 4
+          path: reports-${{github.job}}.tzst
+  # Uncomment these two lines if you need ssh access to debug a build.
+  #      - name: Setup upterm session
+  #        uses: lhotari/action-upterm@v1
+
+
+  unit_test:
+    name: PSC Unit Tests
+    needs: build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Unit tests
+        run: ./gradlew pravega-sensor-collector:test --parallel ${{env.GRADLE_OPTS}}
+      - name: Tar Reports
+        run: tar --use-compress-program zstd -cf reports-${{github.job}}.tzst `echo ${{env.REPORTS_LOCATIONS}}`
+      - name: Upload Reports
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{github.job}}-reports
+          retention-days: 4
+          path: reports-${{github.job}}.tzst
 
-      - name: Set env
-        if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
 
-      - name: Release
-        if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # DO NOT RENAME THIS JOB. Mergers to master branch are gated on this completing successfully.
+  build_and_test_complete:
+    name: CI Complete
+    needs: [build, unit_test]
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check Build Status
+        run: echo Build, static analysis, unit and integration tests successful.
+      - name: Checkout
+        uses: actions/checkout@v2
         with:
-          files: ./pravega-sensor-collector/build/distributions/pravega-sensor-collector-*.tgz
-          draft: true
-          name: pravega-sensor-collector ${{ env.RELEASE_VERSION }}
+          fetch-depth: 2
+      - name: Download code coverage reports
+        uses: actions/download-artifact@v2
+      - name: Untar reports
+        run: ( ls */reports-*.tzst | xargs -n1 tar --use-compress-program zstd --keep-newer-files -xf )
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v1
+

README.md

@@ -164,6 +164,8 @@ For a list of commonly-used configuration values, see the
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_DELETE_COMPLETED_FILES`      |                                                                                                                     `false`                                                                                                                     |                                                                                                                                                         If true, PSC immediately delete the file soon after processing                                                                                                                                                         |
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_TRANSACTION_TIMEOUT_MINUTES` |                                                                                                                      `2.0`                                                                                                                      |                                                                                                                                                            Timeout for each transaction. Default value is 2 minutes                                                                                                                                                            |
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_CREATE_SCOPE`                |                                                                                                                     `false`                                                                                                                     |                                                                                                                                                        If Pravega is on SDP, set this to `false`. Accept Boolean value.                                                                                                                                                        |
+| `PRAVEGA_SENSOR_COLLECTOR_RAW1_EXACTLY_ONCE`                |                                                                                                                      true                                                                                                                       |                                                                                                 If true, it will use transactional write. For raw file ingestion it is recommended to set it as false as in transactional write, client can process maximum file size of 8mb.                                                                                                  |
+| `PRAVEGA_SENSOR_COLLECTOR_RAW1_ENABLE_LARGE_EVENT`          |                                                                                                                      false                                                                                                                      |                                                                                                                                if false, will not allow to write large event. It is recommended to set it as true for non transactional write.                                                                                                                                 |
 | `HADOOP_HOME`                                               |                                                                                                                  `${HOME}/dev`                                                                                                                  | For windows, Hadoop requires native libraries on Windows to work properly. You can download `Winutils.exe` to fix this. <br> See [here](https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems). Add the location of bin/winutils.exe in the parameter HADOOP_HOME.  <br> **This is required only for Parquet file type not for CSV and Raw file ingestion type** |
 
 
@@ -244,9 +246,7 @@ If the TLS Certificate Authority (CA) used by Pravega is not trusted by a well-k
     ```
     OR
     ```
-    kubectl get secret pravega-controller-tls -n nautilus-pravega -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/pravega.crt
-    kubectl get secret keycloak-tls -n nautilus-system -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/keycloak.crt
-    kubectl get secret pravega-tls -n nautilus-pravega -o jsonpath="{.data.tls\.crt}" | base64 --decode > ~/pravegaAll.crt
+    kubectl get cabundle sdp-default -n nautilus-system -o jsonpath="{.spec.certificates.sdp}" > ~/tlsCA-sdp.crt
     ```
 2. On the target system, add the CA certificate to the operating system.
     ```shell

config/checkstyle.xml

@@ -9,7 +9,7 @@
 
   <!-- header -->
   <module name="RegexpHeader">
-    <property name="header" value="/\*\*\n * Copyright Pravega Authors\."/>
+    <property name="header" value="/\*\*\n * Copyright \(c\) Dell Inc., or its subsidiaries. All Rights Reserved."/>
   </module>
   <module name="SuppressionFilter">
         <property name="file" value="${suppressionsFile}"/>
@@ -54,7 +54,7 @@
     <module name="LeftCurly"/>
     <module name="RightCurly"/>
     <module name="ConstantName">
-      <property name="format" value="(^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$)|(^log$)"/>
+      <property name="format" value="(^[A-Za-z][A-Za-z0-9]*(_[A-Z0-9]+)*$)|(^log$)"/>
     </module>
     <module name="LocalVariableName"/>
     <module name="LocalFinalVariableName"/>

config/spotbugs-exclude.xml

@@ -29,9 +29,52 @@
     <Match> <!-- does not work from JDK11 onwards https://github.com/spotbugs/spotbugs/issues/756 -->
         <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE" />
     </Match>
+    <Match> <!-- does not work from JDK11 onwards https://github.com/spotbugs/spotbugs/issues/756 -->
+        <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+    </Match>
+
     <Match> <!-- Returning a java.lang.Boolean is common with Streams/Lambdas -->
         <Bug pattern="NP_BOOLEAN_RETURN_NULL" />
     </Match>
+    <Match>
+        <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.util.TransactionCoordinator" />
+        <Bug pattern="CT_CONSTRUCTOR_THROW" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.file.csvfile.CsvFileSequenceProcessor" />
+        <Bug pattern="UR_UNINIT_READ_CALLED_FROM_SUPER_CONSTRUCTOR" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.file.parquet.ParquetFileProcessor" />
+        <Bug pattern="UR_UNINIT_READ_CALLED_FROM_SUPER_CONSTRUCTOR" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.file.rawfile.RawFileProcessor" />
+        <Bug pattern="UR_UNINIT_READ_CALLED_FROM_SUPER_CONSTRUCTOR" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.util.TransactionCoordinatorTests" />
+        <Bug pattern="ODR_OPEN_DATABASE_RESOURCE" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.accelerometer.AccelerometerSamples" />
+        <Bug pattern="PA_PUBLIC_PRIMITIVE_ATTRIBUTE" />
+    </Match>
+
+    <Match>
+        <Class name="io.pravega.sensor.collector.network.NetworkSamples" />
+        <Bug pattern="PA_PUBLIC_PRIMITIVE_ATTRIBUTE" />
+    </Match>
+
     <Match> <!-- This is intended in these and only these classes -->
         <Or>
             <Bug pattern="EI_EXPOSE_REP" />

config/spotbugs-include.xml

@@ -8,9 +8,9 @@
 
     <!-- A use of untrusted input in a way that could create a remotely exploitable 
         security vulnerability. -->
-    <Match>
+    <!-- <Match>
         <Bug category="SECURITY" />
-    </Match>
+    </Match> -->
 
     <!-- Violations of recommended and essential coding practice. Examples include 
         hash code and equals problems, cloneable idiom, dropped exceptions, Serializable 
@@ -39,9 +39,9 @@
     </Match>
 
     <!-- code flaws having to do with threads, locks, and volatiles -->
-    <Match>
+    <!-- <Match>
         <Bug category="MT_CORRECTNESS" />
-    </Match>
+    </Match> -->
 
     <!-- Experimental and not fully vetted bug patterns -->
     <Match>

gradle.properties

@@ -13,11 +13,12 @@ commonsCLIVersion=1.4
 commonsCSVVersion=1.8
 commonsCodecVersion=1.14
 commonsMath3Version=3.6.1
-grizzlyVersion=3.1.3
+grizzlyVersion=2.35
 gsonVersion=2.10.1
 includePravegaCredentials=true
 jacksonVersion=2.15.2
-junitVersion=5.6.2
+junitVersion=5.10.1
+junitPlatformVersion=1.10.1
 jakartaBindVersion=2.3.2
 jaxbVersion=2.3.2
 javaxServletApiVersion=3.0.1

parquet-file-sample-data/integration-test/hello-world.parquet

@@ -0,0 +1 @@
+Hello World.

parquet-file-sample-data/test_file/hello-world.parquet

@@ -0,0 +1 @@
+Hello World.

pravega-sensor-collector/build.gradle

@@ -46,7 +46,10 @@ dependencies {
 
     implementation "io.pravega:pravega-client:${pravegaVersion}",
             "io.pravega:pravega-common:${pravegaVersion}",
-            "commons-cli:commons-cli:${commonsCLIVersion}"
+            "commons-cli:commons-cli:${commonsCLIVersion}",
+            "io.pravega:pravega-standalone:${pravegaVersion}",
+            "io.pravega:pravega-test-integration:${pravegaVersion}"
+
 
     if (includePravegaCredentials.toBoolean()) {
         implementation "io.pravega:pravega-keycloak-client:${pravegaCredentialsVersion}"
@@ -68,7 +71,7 @@ dependencies {
     testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
     testImplementation "org.junit.vintage:junit-vintage-engine:${junitVersion}"
     testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
-    testImplementation "org.junit.platform:junit-platform-launcher"
+    testImplementation "org.junit.platform:junit-platform-launcher:${junitPlatformVersion}"
 
     testImplementation "org.mockito:mockito-core:${mockitoVersion}"
 
@@ -105,6 +108,7 @@ startScripts {
 }
 
 shadowJar{
+    zip64 true
     archiveBaseName = 'pravega-sensor-collector'
     archiveClassifier = ''
 }

pravega-sensor-collector/src/main/dist/conf/env-sample-network.sh

@@ -10,7 +10,7 @@
 export pravega_client_auth_method=Bearer
 export pravega_client_auth_loadDynamic=true
 export KEYCLOAK_SERVICE_ACCOUNT_FILE=/opt/pravega-sensor-collector/conf/keycloak.json
-export JAVA_OPTS="-Xmx512m"
+export JAVA_OPTS="-Xmx512m -Dlogback.configurationFile=/opt/pravega-sensor-collector/bin/logback.xml"
 export PRAVEGA_SENSOR_COLLECTOR_NET1_CLASS=io.pravega.sensor.collector.network.NetworkDriver
 export PRAVEGA_SENSOR_COLLECTOR_NET1_NETWORK_INTERFACE=ens33
 export PRAVEGA_SENSOR_COLLECTOR_NET1_MEMORY_QUEUE_CAPACITY_ELEMENTS=10000

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriver.java

@@ -9,6 +9,7 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.AbstractService;
 import io.pravega.client.ClientConfig;
 import io.pravega.client.EventStreamClientFactory;
@@ -30,7 +31,7 @@ public abstract class DeviceDriver extends AbstractService implements AutoClosea
     private static final String CREATE_SCOPE_KEY = "CREATE_SCOPE";
 
     public DeviceDriver(DeviceDriverConfig config) {
-        this.config = config;
+        this.config = Preconditions.checkNotNull(config, "config");
         LOGGER.info("Create Scope: {}", isCreateScope());
     }
 

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriverConfig.java

@@ -9,6 +9,8 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
+
 import java.util.Map;
 
 public class DeviceDriverConfig {
@@ -18,10 +20,10 @@ public class DeviceDriverConfig {
     private final DeviceDriverManager deviceDriverManager;
 
     public DeviceDriverConfig(String instanceName, String className, Map<String, String> properties, DeviceDriverManager deviceDriverManager) {
-        this.instanceName = instanceName;
-        this.className = className;
-        this.properties = properties;
-        this.deviceDriverManager = deviceDriverManager;
+        this.instanceName = Preconditions.checkNotNull(instanceName, "instanceName");
+        this.className = Preconditions.checkNotNull(className, "className");
+        this.properties = Preconditions.checkNotNull(properties, "deviceDriverProperties");
+        this.deviceDriverManager = Preconditions.checkNotNull(deviceDriverManager, "deviceDriverManager");
     }
 
     @Override

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriverFactory.java

@@ -9,6 +9,7 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,6 +21,7 @@ public class DeviceDriverFactory {
      */
     DeviceDriver create(DeviceDriverConfig config) {
         try {
+            Preconditions.checkNotNull(config, "deviceDriverConfig");
             LOGGER.info("Creating device driver instance {} with class {}", config.getInstanceName(), config.getClassName());
             final Class<?> deviceDriverClass = Class.forName(config.getClassName());
             final DeviceDriver driver = (DeviceDriver) deviceDriverClass.getConstructor(DeviceDriverConfig.class).newInstance(config);

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriverManager.java

@@ -9,6 +9,7 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.AbstractService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,25 +32,28 @@ public class DeviceDriverManager extends AbstractService {
     private List<DeviceDriver> drivers;
 
     public DeviceDriverManager(Map<String, String> properties) {
-        configs = configFromProperties(PREFIX, SEPARATOR, properties);
+        configs = configFromProperties(PREFIX, SEPARATOR, Preconditions.checkNotNull(properties, "properties"));
     }
 
     @Override
     protected void doStart() {
         LOGGER.info("Starting device drivers");
         final DeviceDriverFactory factory = new DeviceDriverFactory();
         drivers = configs.stream().map(factory::create).collect(Collectors.toList());
-        drivers.stream().forEach((driver) -> driver.startAsync());
-        drivers.stream().forEach((driver) -> driver.awaitRunning());
+        drivers.forEach(AbstractService::startAsync);
+        drivers.forEach(AbstractService::awaitRunning);
         LOGGER.info("All device drivers started successfully");
         notifyStarted();
     }
 
     @Override
     protected void doStop() {
-        drivers.stream().forEach((driver) -> driver.stopAsync());
-        drivers.stream().forEach((driver) -> driver.awaitTerminated());
+        LOGGER.info("Stopping all device drivers");
+        drivers.forEach(AbstractService::stopAsync);
+        drivers.forEach(AbstractService::awaitTerminated);
         drivers = null;
+        LOGGER.info("Stopped all device drivers");
+        notifyStopped();
     }
 
     /**