Karman 10882 enable GitHub checks

.codecov.yml

@@ -0,0 +1,33 @@
+#
+# Copyright Pravega Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+codecov:
+  require_ci_to_pass: yes
+  max_report_age: off
+  notify:
+    wait_for_ci: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 0.5%
+    patch:
+      default:
+        target: 80%
+  ignore:
+    - "**/generated/**"
+    - "standalone"
+    - "test"

.github/workflows/gradle.yml

@@ -0,0 +1,154 @@
+#
+# Copyright (c) Dell Inc., or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+name: build
+
+on: [push, pull_request, workflow_dispatch]
+  # workflow_dispatch should make manually triggered ci/cd possible
+  # workflow file (like this) with `workflow_dispatch` after on should exist on the **master** or default branch,
+  # or there will be no ui for a manual trigger. https://github.community/t/workflow-dispatch-event-not-working/128856/2
+
+  #
+  # Set 'GRADLE_OPTS' to pass additional custom parameters to each ./gradlew invocation in this workflow.
+  # Example '--info' or '--debug'.
+  #GRADLE_OPTS: --info
+
+# The workflow begins with a compilation and static analysis job that also caches the build output and source code,
+# followed by a number of parallel test jobs (which make use of that cached artifacts).
+#
+# Once the build job and all the test jobs complete successfully, a final (no-op) job ("build_and_test_complete") will
+# automatically complete. This job must NOT be renamed as the GitHub Pravega Repository gates merges into master on
+# this step passing.
+#
+# Finally, a "snapshot" job is triggered only for pushes (commits) to master and release branches, which publishes all
+# artifacts to a public repository.
+
+env:
+  GLOBAL_CACHE_PATH: |
+    ~/.gradle
+    ~/.m2
+  GLOBAL_CACHE_KEY: gradle-m2-java-11
+  GLOBAL_CACHE_RESTORE_KEYS: |
+    gradle-m2-java-
+    gradle-m2
+  # We cache the class files, resources and build output. This is generated on the build job and reused in dependent jobs.
+  # If there are subsequent builds this includes test execution.
+  BUILD_CACHE_PATH: |
+    .gradle
+    ./bin
+    **/bin
+    **/build
+
+  REPORTS_LOCATIONS: |
+    pravega-sensor-collector/build/reports
+
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Build Information
+        run: echo Building a '${{ github.event_name }}' for target '${{ github.ref }}'.
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Test script
+        run: ./scripts/build-installer.sh
+      - name: Clean
+        if: ${{ startsWith(github.ref, 'refs/heads/') }}
+        run: ./gradlew clean ${{env.GRADLE_OPTS}}
+      - name: Compile & Checkstyle
+        run: ./gradlew jar compileTest checkstyleMain checkstyleTest --parallel ${{env.GRADLE_OPTS}}
+      - name: Spotbugs
+        run: ./gradlew spotbugsMain spotbugsTest
+      - name: Tar Reports
+        run: tar --use-compress-program zstd -cf reports-${{github.job}}.tzst `echo ${{env.REPORTS_LOCATIONS}}`
+      - name: Upload Reports
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{github.job}}-reports
+          retention-days: 4
+          path: reports-${{github.job}}.tzst
+# Uncomment these two lines if you need ssh access to debug a build.
+#      - name: Setup upterm session
+#        uses: lhotari/action-upterm@v1
+
+
+  unit_test:
+    name: PSC Unit Tests
+    needs: build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Unit tests
+        run: ./gradlew build --parallel ${{env.GRADLE_OPTS}}
+      - name: PWD
+        run: pwd
+      - name: LS
+        run: ls
+      - name: LS PSC/build
+        run: ls pravega-sensor-collector/build
+      - name: Tar Reports
+        run: tar --use-compress-program zstd -cf reports-${{github.job}}.tzst `echo ${{env.REPORTS_LOCATIONS}}`
+      - name: Upload Reports
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{github.job}}-reports
+          retention-days: 4
+          path: reports-${{github.job}}.tzst
+
+
+  # DO NOT RENAME THIS JOB. Mergers to master branch are gated on this completing successfully.
+  build_and_test_complete:
+    name: CI Complete
+    needs: [build, unit_test]
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check Build Status
+        run: echo Build, static analysis, unit and integration tests successful.
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Download code coverage reports
+        uses: actions/download-artifact@v2
+      - name: Untar reports
+        run: ( ls */reports-*.tzst | xargs -n1 tar --use-compress-program zstd --keep-newer-files -xf )
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v1
+
+  snapshot:
+    name: Publish snapshot packages
+    needs: [build_and_test_complete]
+    # Only run this on PUSH (no pull requests) and only on the master branch and release branches.
+    if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/r0.') || startsWith(github.ref, 'refs/heads/r1.')) }}
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Assemble
+        run: ./gradlew assemble --parallel ${{env.GRADLE_OPTS}}
+      - name: Publish to GitHub Packages
+        run: ./gradlew publish -PpublishUrl=https://maven.pkg.github.com/${{github.repository}} -PpublishUsername=${{github.actor}} -PpublishPassword=${{secrets.GITHUB_TOKEN}} ${{env.GRADLE_OPTS}}

README.md

@@ -164,6 +164,8 @@ For a list of commonly-used configuration values, see the
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_DELETE_COMPLETED_FILES`      |                                                                                                                     `false`                                                                                                                     |                                                                                                                                                         If true, PSC immediately delete the file soon after processing                                                                                                                                                         |
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_TRANSACTION_TIMEOUT_MINUTES` |                                                                                                                      `2.0`                                                                                                                      |                                                                                                                                                            Timeout for each transaction. Default value is 2 minutes                                                                                                                                                            |
 | `PRAVEGA_SENSOR_COLLECTOR_RAW1_CREATE_SCOPE`                |                                                                                                                     `false`                                                                                                                     |                                                                                                                                                        If Pravega is on SDP, set this to `false`. Accept Boolean value.                                                                                                                                                        |
+| `PRAVEGA_SENSOR_COLLECTOR_RAW1_EXACTLY_ONCE`                |                                                                                                                      true                                                                                                                       |                                                                                                 If true, it will use transactional write. For raw file ingestion it is recommended to set it as false as in transactional write, client can process maximum file size of 8mb.                                                                                                  |
+| `PRAVEGA_SENSOR_COLLECTOR_RAW1_ENABLE_LARGE_EVENT`          |                                                                                                                      false                                                                                                                      |                                                                                                                                if false, will not allow to write large event. It is recommended to set it as true for non transactional write.                                                                                                                                 |
 | `HADOOP_HOME`                                               |                                                                                                                  `${HOME}/dev`                                                                                                                  | For windows, Hadoop requires native libraries on Windows to work properly. You can download `Winutils.exe` to fix this. <br> See [here](https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems). Add the location of bin/winutils.exe in the parameter HADOOP_HOME.  <br> **This is required only for Parquet file type not for CSV and Raw file ingestion type** |
 
 

pravega-sensor-collector/.codecov.yml

@@ -0,0 +1,33 @@
+#
+# Copyright Pravega Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+codecov:
+  require_ci_to_pass: yes
+  max_report_age: off
+  notify:
+    wait_for_ci: yes
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 0.5%
+    patch:
+      default:
+        target: 80%
+  ignore:
+    - "**/generated/**"
+    - "standalone"
+    - "test"

pravega-sensor-collector/build.gradle

@@ -26,6 +26,7 @@ apply plugin: "com.github.spotbugs"
 apply plugin: 'checkstyle'
 apply plugin: "application"
 apply plugin: "com.github.johnrengelman.shadow"
+apply plugin: "jacoco"
 
 group = "io.pravega"
 archivesBaseName = "pravega-sensor-collector"
@@ -156,3 +157,17 @@ checkstyle {
     }
 }
 
+
+plugins.withId('jacoco') {
+    jacoco {
+        toolVersion = "0.8.5"
+    }
+    jacocoTestReport {
+        reports {
+            xml.enabled true
+            html.enabled false
+        }
+    }
+
+    test { finalizedBy jacocoTestReport }
+}

pravega-sensor-collector/src/main/dist/conf/env-sample-network.sh

@@ -10,7 +10,7 @@
 export pravega_client_auth_method=Bearer
 export pravega_client_auth_loadDynamic=true
 export KEYCLOAK_SERVICE_ACCOUNT_FILE=/opt/pravega-sensor-collector/conf/keycloak.json
-export JAVA_OPTS="-Xmx512m"
+export JAVA_OPTS="-Xmx512m -Dlogback.configurationFile=/opt/pravega-sensor-collector/bin/logback.xml"
 export PRAVEGA_SENSOR_COLLECTOR_NET1_CLASS=io.pravega.sensor.collector.network.NetworkDriver
 export PRAVEGA_SENSOR_COLLECTOR_NET1_NETWORK_INTERFACE=ens33
 export PRAVEGA_SENSOR_COLLECTOR_NET1_MEMORY_QUEUE_CAPACITY_ELEMENTS=10000

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriver.java

@@ -9,6 +9,7 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.AbstractService;
 import io.pravega.client.ClientConfig;
 import io.pravega.client.EventStreamClientFactory;
@@ -23,15 +24,15 @@
  * This is an abstract class for all device drivers.
  */
 public abstract class DeviceDriver extends AbstractService implements AutoCloseable {
-    private static final Logger log = LoggerFactory.getLogger(DeviceDriver.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(DeviceDriver.class);
 
     private final DeviceDriverConfig config;
 
     private static final String CREATE_SCOPE_KEY = "CREATE_SCOPE";
 
     public DeviceDriver(DeviceDriverConfig config) {
-        this.config = config;
-        log.info("Create Scope: {}", isCreateScope());
+        this.config = Preconditions.checkNotNull(config, "config");
+        LOGGER.info("Create Scope: {}", isCreateScope());
     }
 
     public String getProperty(String key) {

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriverConfig.java

@@ -9,6 +9,8 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
+
 import java.util.Map;
 
 public class DeviceDriverConfig {
@@ -18,19 +20,19 @@ public class DeviceDriverConfig {
     private final DeviceDriverManager deviceDriverManager;
 
     public DeviceDriverConfig(String instanceName, String className, Map<String, String> properties, DeviceDriverManager deviceDriverManager) {
-        this.instanceName = instanceName;
-        this.className = className;
-        this.properties = properties;
-        this.deviceDriverManager = deviceDriverManager;
+        this.instanceName = Preconditions.checkNotNull(instanceName, "instanceName");
+        this.className = Preconditions.checkNotNull(className, "className");
+        this.properties = Preconditions.checkNotNull(properties, "deviceDriverProperties");
+        this.deviceDriverManager = Preconditions.checkNotNull(deviceDriverManager, "deviceDriverManager");
     }
 
     @Override
     public String toString() {
-        return "DeviceDriverConfig{" +
-                "instanceName='" + instanceName + '\'' +
-                ", className='" + className + '\'' +
-                ", properties=" + properties +
-                '}';
+        return "DeviceDriverConfig{"
+                + "instanceName='" + instanceName + '\''
+                + ", className='" + className + '\''
+                + ", properties=" + properties
+                + '}';
     }
 
     public String getInstanceName() {

pravega-sensor-collector/src/main/java/io/pravega/sensor/collector/DeviceDriverFactory.java

@@ -9,21 +9,23 @@
  */
 package io.pravega.sensor.collector;
 
+import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class DeviceDriverFactory {
-    private static final Logger log = LoggerFactory.getLogger(DeviceDriverFactory.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(DeviceDriverFactory.class);
 
     /**
      * Instantiate a concrete subclass of DeviceDriver based on key/value properties.
      */
     DeviceDriver create(DeviceDriverConfig config) {
         try {
-            log.info("Creating device driver instance {} with class {}", config.getInstanceName(), config.getClassName());
+            Preconditions.checkNotNull(config, "deviceDriverConfig");
+            LOGGER.info("Creating device driver instance {} with class {}", config.getInstanceName(), config.getClassName());
             final Class<?> deviceDriverClass = Class.forName(config.getClassName());
             final DeviceDriver driver = (DeviceDriver) deviceDriverClass.getConstructor(DeviceDriverConfig.class).newInstance(config);
-            log.info("Done creating device driver instance {}", config.getInstanceName());
+            LOGGER.info("Done creating device driver instance {}", config.getInstanceName());
             return driver;
         } catch (Exception e) {
             throw new RuntimeException(e);