Transformer: Add TextGenPipeline + FeaturizePipeline

[Lundez]

Some code I did during first implementation of Transformers,

class FeatureizeTranslator(val tokenizer: HuggingFaceTokenizerWrapper) :
    Translator<String, NDArray> {
    override fun processInput(ctx: TranslatorContext, input: String): NDList {
        val inputEncoding = tokenizer.encode(input)
        val ids = ctx.ndManager.create(inputEncoding.ids)
        val attention = ctx.ndManager.create(inputEncoding.attentionMask)

        return NDList(ids, attention)
    }

    override fun processOutput(ctx: TranslatorContext, list: NDList): NDArray {
        return list[0]
    }
}
class FeaturizePipeline(model: Criteria<String, NDArray>) : TransformerPipeline<String, NDArray>(model) {
    companion object {
        fun create(
            modelName: String,
            engine: Engine = Engine.ONNX,
            device: Device = Device.cpu(),
        ): TransformerPipeline<String, NDArray> {
            val model = HuggingFaceModelHub.load(modelName, engine)
            val translator = FeatureizeTranslator(HuggingFaceTokenizerWrapper(modelName))
            val criteria = baseCriteria(translator, model.localPath, engine, device)

            return FeaturizePipeline(criteria)
        }

        @JvmStatic
        fun main(args: Array<String>) {
            create("optimum/bert-base-NER").use { pipeline ->
                println(pipeline.predict("My name is Sarah and I live in London"))
            }
        }
    }
}