diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 075e8d6662..c493113f46 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -213,6 +213,7 @@ "URL Decode", "Protobuf Decode", "Protobuf Encode", + "Protobuf Stream Decode", "VarInt Encode", "VarInt Decode", "JA3 Fingerprint", diff --git a/src/core/lib/Protobuf.mjs b/src/core/lib/Protobuf.mjs index e131d3a527..e5cae4fc33 100644 --- a/src/core/lib/Protobuf.mjs +++ b/src/core/lib/Protobuf.mjs @@ -116,6 +116,30 @@ class Protobuf { return this.mergeDecodes(input); } + /** + * Parse Protobuf stream data + * @param {byteArray} input + * @param {any[]} args + * @returns {any[]} + */ + static decodeStream(input, args) { + this.updateProtoRoot(args[0]); + this.showUnknownFields = args[1]; + this.showTypes = args[2]; + + const streams = new Protobuf(input); + const output = []; + let objLength = streams._varInt(); + while (!isNaN(objLength) && objLength > 0) { + const subData = streams.data.slice(streams.offset, streams.offset + objLength); + output.push(this.mergeDecodes(subData)); + streams.offset += objLength; + objLength = streams._varInt(); + } + + return output; + } + /** * Update the parsedProto, throw parsing errors * diff --git a/src/core/operations/ProtobufStreamDecode.mjs b/src/core/operations/ProtobufStreamDecode.mjs new file mode 100644 index 0000000000..01243b54b7 --- /dev/null +++ b/src/core/operations/ProtobufStreamDecode.mjs @@ -0,0 +1,54 @@ +/** + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import Protobuf from "../lib/Protobuf.mjs"; + +/** + * Protobuf Stream Decode operation + */ +class ProtobufStreamDecode extends Operation { + + /** + * ProtobufStreamDecode constructor + */ + constructor() { + super(); + + this.name = "Protobuf Stream Decode"; + this.module = "Protobuf"; + this.description = "Decodes Protobuf encoded data from streams to a JSON array representation of the data using the field number as the field key.

If a .proto schema is defined, the encoded data will be decoded with reference to the schema. Only one message instance will be decoded.

Show Unknown Fields
When a schema is used, this option shows fields that are present in the input data but not defined in the schema.

Show Types
Show the type of a field next to its name. For undefined fields, the wiretype and example types are shown instead."; + this.infoURL = "https://developers.google.com/protocol-buffers/docs/techniques#streaming"; + this.inputType = "ArrayBuffer"; + this.outputType = "JSON"; + this.args = [ + { + name: "Show Types", + type: "boolean", + value: false + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + input = new Uint8Array(input); + try { + // provide standard values for currently removed arguments + return Protobuf.decodeStream(input, ["", false, ...args]); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default ProtobufStreamDecode; diff --git a/tests/operations/tests/Protobuf.mjs b/tests/operations/tests/Protobuf.mjs index 2131e72340..3ac2051872 100644 --- a/tests/operations/tests/Protobuf.mjs +++ b/tests/operations/tests/Protobuf.mjs @@ -303,4 +303,33 @@ TestRegister.addTests([ } ] }, + { + name: "Protobuf Stream Decode: no schema", + input: "0d081c1203596f751a024d65202b0c0a0a0a066162633132331200", + expectedOutput: JSON.stringify([ + { + "1": 28, + "2": "You", + "3": "Me", + "4": 43 + }, + { + "1": { + "1": "abc123", + "2": {} + } + } + ], null, 4), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Auto"] + }, + { + "op": "Protobuf Stream Decode", + "args": [false] + } + ] + }, + ]);