add Protobuf Stream Decode operation

Author: oliverrahner

Diff for: src/core/config/Categories.json

@@ -207,6 +207,7 @@
             "URL Decode",
             "Protobuf Decode",
             "Protobuf Encode",
+            "Protobuf Stream Decode",
             "VarInt Encode",
             "VarInt Decode",
             "JA3 Fingerprint",

Diff for: src/core/lib/Protobuf.mjs

@@ -116,6 +116,24 @@ class Protobuf {
         return this.mergeDecodes(input);
     }
 
+    static decodeStream(input, args) {
+        this.updateProtoRoot(args[0]);
+        this.showUnknownFields = args[1];
+        this.showTypes = args[2];
+
+        let streams = new Protobuf(input);
+        let output = [];
+        let objLength = streams._varInt();
+        while (!isNaN(objLength) && objLength > 0) {
+            let subData = streams.data.slice(streams.offset, streams.offset + objLength);
+            output.push(this.mergeDecodes(subData));
+            streams.offset += objLength;
+            objLength = streams._varInt();
+        }
+
+        return output;
+    }
+
     /**
      * Update the parsedProto, throw parsing errors
      *
@@ -184,7 +202,7 @@ class Protobuf {
                 bytes: String,
                 longs: Number,
                 enums: String,
-                defualts: true
+                defaults: true
             });
             const output = {};
 

Diff for: src/core/operations/ProtobufStreamDecode.mjs

@@ -0,0 +1,54 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
+import Protobuf from "../lib/Protobuf.mjs";
+
+/**
+ * Protobuf Stream Decode operation
+ */
+class ProtobufStreamDecode extends Operation {
+
+    /**
+     * ProtobufStreamDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Protobuf Stream Decode";
+        this.module = "Protobuf";
+        this.description = "Decodes Protobuf encoded data from streams to a JSON array representation of the data using the field number as the field key.<br><br>If a .proto schema is defined, the encoded data will be decoded with reference to the schema. Only one message instance will be decoded. <br><br><u>Show Unknown Fields</u><br>When a schema is used, this option shows fields that are present in the input data but not defined in the schema.<br><br><u>Show Types</u><br>Show the type of a field next to its name. For undefined fields, the wiretype and example types are shown instead.";
+        this.infoURL = "https://developers.google.com/protocol-buffers/docs/techniques#streaming";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "JSON";
+        this.args = [
+            {
+                name: "Show Types",
+                type: "boolean",
+                value: false
+            }
+        ];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        input = new Uint8Array(input);
+        try {
+            // provide standard values for currently removed arguments
+            return Protobuf.decodeStream(input, ["", false, ...args]);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default ProtobufStreamDecode;

Diff for: tests/operations/tests/Protobuf.mjs

@@ -303,4 +303,31 @@ TestRegister.addTests([
             }
         ]
     },
+    {
+        name: "Protobuf Stream Decode: no schema",
+        input: "0d081c1203596f751a024d65202b0c0a0a0a066162633132331200",
+        expectedOutput: JSON.stringify([{
+            "1": 28,
+            "2": "You",
+            "3": "Me",
+            "4": 43
+        },
+        {
+            "1": {
+                "1": "abc123",
+                "2": {}
+            }
+        }], null, 4),
+        recipeConfig: [
+            {
+                "op": "From Hex",
+                "args": ["Auto"]
+            },
+            {
+                "op": "Protobuf Stream Decode",
+                "args": ["", false, false]
+            }
+        ]
+    },
+
 ]);