Added support for spring spel expressions in profiles.

fhermansson · fhermansson · commit 1897aaa9278e · 2024-02-21T09:13:52.000+01:00
See https://docs.spring.io/spring-framework/reference/core/expressions.html. Profiles are parsed as spel expression templates with EncoreJob as root object. Added field profileParams to EncoreJob to be used in profile templates.
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/config/ProfileProperties.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/config/ProfileProperties.kt
@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2020 Sveriges Television AB
+//
+// SPDX-License-Identifier: EUPL-1.2
+
+package se.svt.oss.encore.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.core.io.Resource
+
+@ConfigurationProperties("profile")
+data class ProfileProperties(
+    val location: Resource,
+    val spelExpressionPrefix: String = "#{",
+    val spelExpressionSuffix: String = "}",
+)
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/model/EncoreJob.kt
@@ -7,6 +7,11 @@ package se.svt.oss.encore.model
 import com.fasterxml.jackson.annotation.JsonIgnore
 import io.swagger.v3.oas.annotations.media.Schema
 import io.swagger.v3.oas.annotations.tags.Tag
+import jakarta.validation.constraints.Max
+import jakarta.validation.constraints.Min
+import jakarta.validation.constraints.NotBlank
+import jakarta.validation.constraints.NotEmpty
+import jakarta.validation.constraints.Positive
 import org.springframework.data.annotation.Id
 import org.springframework.data.redis.core.RedisHash
 import org.springframework.data.redis.core.index.Indexed
@@ -15,11 +20,6 @@ import se.svt.oss.encore.model.input.Input
 import se.svt.oss.mediaanalyzer.file.MediaFile
 import java.time.OffsetDateTime
 import java.util.UUID
-import jakarta.validation.constraints.Max
-import jakarta.validation.constraints.Min
-import jakarta.validation.constraints.NotBlank
-import jakarta.validation.constraints.NotEmpty
-import jakarta.validation.constraints.Positive
 
 @Validated
 @RedisHash("encore-jobs", timeToLive = (60 * 60 * 24 * 7).toLong()) // 1 week ttl
@@ -51,6 +51,12 @@ data class EncoreJob(
     @NotBlank
     val profile: String,
 
+    @Schema(
+        description = "Properties for evaluation of spring spel expressions in profile",
+        defaultValue = "{}"
+    )
+    val profileParams: Map<String, Any?> = emptyMap(),
+
     @Schema(
         description = "A directory path to where the output should be written",
         example = "/an/output/path/dir",
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/FfmpegExecutor.kt
@@ -42,7 +42,7 @@ class FfmpegExecutor(
         outputFolder: String,
         progressChannel: SendChannel<Int>?
     ): List<MediaFile> {
-        val profile = profileService.getProfile(encoreJob.profile)
+        val profile = profileService.getProfile(encoreJob)
         val outputs = profile.encodes.mapNotNull {
             it.getOutput(
                 encoreJob,
diff --git a/encore-common/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt b/encore-common/src/main/kotlin/se/svt/oss/encore/service/profile/ProfileService.kt
@@ -11,9 +11,14 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import mu.KotlinLogging
 import org.springframework.aot.hint.annotation.RegisterReflectionForBinding
-import org.springframework.beans.factory.annotation.Value
-import org.springframework.core.io.Resource
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.expression.common.TemplateParserContext
+import org.springframework.expression.spel.SpelParserConfiguration
+import org.springframework.expression.spel.standard.SpelExpressionParser
+import org.springframework.expression.spel.support.SimpleEvaluationContext
 import org.springframework.stereotype.Service
+import se.svt.oss.encore.config.ProfileProperties
+import se.svt.oss.encore.model.EncoreJob
 import se.svt.oss.encore.model.profile.AudioEncode
 import se.svt.oss.encore.model.profile.GenericVideoEncode
 import se.svt.oss.encore.model.profile.OutputProducer
@@ -38,35 +43,65 @@ import java.util.Locale
     ThumbnailEncode::class,
     ThumbnailMapEncode::class
 )
+@EnableConfigurationProperties(ProfileProperties::class)
 class ProfileService(
-    @Value("\${profile.location}")
-    private val profileLocation: Resource,
+    private val properties: ProfileProperties,
     objectMapper: ObjectMapper
 ) {
     private val log = KotlinLogging.logger { }
 
+    private val spelExpressionParser = SpelExpressionParser(
+        SpelParserConfiguration(
+            null,
+            null,
+            false,
+            false,
+            Int.MAX_VALUE,
+            100_000
+        )
+    )
+
+    private val spelEvaluationContext = SimpleEvaluationContext
+        .forReadOnlyDataBinding()
+        .build()
+
+    private val spelParserContext = TemplateParserContext(
+        properties.spelExpressionPrefix,
+        properties.spelExpressionSuffix
+    )
+
     private val mapper =
-        if (profileLocation.filename?.let { File(it).extension.lowercase(Locale.getDefault()) in setOf("yml", "yaml") } == true) {
+        if (properties.location.filename?.let {
+            File(it).extension.lowercase(Locale.getDefault()) in setOf(
+                    "yml",
+                    "yaml"
+                )
+        } == true
+        ) {
             yamlMapper()
         } else {
             objectMapper
         }
 
-    fun getProfile(name: String): Profile = try {
-        log.debug { "Get profile $name. Reading profiles from $profileLocation" }
-        val profiles = mapper.readValue<Map<String, String>>(profileLocation.inputStream)
+    fun getProfile(job: EncoreJob): Profile = try {
+        log.debug { "Get profile ${job.profile}. Reading profiles from ${properties.location}" }
+        val profiles = mapper.readValue<Map<String, String>>(properties.location.inputStream)
 
-        profiles[name]
-            ?.let { readProfile(it) }
-            ?: throw RuntimeException("Could not find location for profile $name! Profiles: $profiles")
+        profiles[job.profile]
+            ?.let { readProfile(it, job) }
+            ?: throw RuntimeException("Could not find location for profile ${job.profile}! Profiles: $profiles")
     } catch (e: JsonProcessingException) {
-        throw RuntimeException("Error parsing profile $name: ${e.message}", e)
+        throw RuntimeException("Error parsing profile ${job.profile}: ${e.message}", e)
     }
 
-    private fun readProfile(path: String): Profile {
-        val profile = profileLocation.createRelative(path)
+    private fun readProfile(path: String, job: EncoreJob): Profile {
+        val profile = properties.location.createRelative(path)
         log.debug { "Reading $profile" }
-        return mapper.readValue(profile.inputStream)
+        val profileContent = profile.inputStream.bufferedReader().use { it.readText() }
+        val resolvedProfileContent = spelExpressionParser
+            .parseExpression(profileContent, spelParserContext)
+            .getValue(spelEvaluationContext, job) as String
+        return mapper.readValue(resolvedProfileContent)
     }
 
     private fun yamlMapper() =
diff --git a/encore-common/src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt b/encore-common/src/test/kotlin/se/svt/oss/encore/service/profile/ProfileServiceTest.kt
@@ -9,7 +9,11 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.springframework.core.io.ClassPathResource
+import se.svt.oss.encore.Assertions.assertThat
 import se.svt.oss.encore.Assertions.assertThatThrownBy
+import se.svt.oss.encore.config.ProfileProperties
+import se.svt.oss.encore.defaultEncoreJob
+import se.svt.oss.encore.model.profile.GenericVideoEncode
 import java.io.IOException
 
 class ProfileServiceTest {
@@ -19,50 +23,67 @@ class ProfileServiceTest {
 
     @BeforeEach
     internal fun setUp() {
-        profileService = ProfileService(ClassPathResource("profile/profiles.yml"), objectMapper)
+        profileService = ProfileService(ProfileProperties(ClassPathResource("profile/profiles.yml")), objectMapper)
     }
 
     @Test
     fun `successfully parses valid yaml profiles`() {
-        listOf("archive", "program-x265", "program").forEach {
-            profileService.getProfile(it)
+        listOf("program-x265", "program").forEach {
+            profileService.getProfile(jobWithProfile(it))
         }
     }
 
+    @Test
+    fun `successully uses profile params`() {
+        val profile = profileService.getProfile(
+            jobWithProfile("archive").copy(
+                profileParams = mapOf("height" to 1080, "suffix" to "test_suffix")
+            )
+        )
+        assertThat(profile.encodes).describedAs("encodes").hasSize(1)
+        val outputProducer = profile.encodes.first()
+        assertThat(outputProducer).isInstanceOf(GenericVideoEncode::class.java)
+        assertThat(outputProducer as GenericVideoEncode)
+            .hasHeight(1080)
+            .hasSuffix("test_suffix")
+    }
+
     @Test
     fun `invalid yaml throws exception`() {
-        assertThatThrownBy { profileService.getProfile("test-invalid") }
+        assertThatThrownBy { profileService.getProfile(jobWithProfile("test-invalid")) }
             .isInstanceOf(RuntimeException::class.java)
             .hasCauseInstanceOf(JsonProcessingException::class.java)
             .hasMessageStartingWith("Error parsing profile test-invalid: Instantiation of [simple type, class se.svt.oss.encore.model.profile.X264Encode] value failed")
     }
 
     @Test
     fun `unknown profile throws error`() {
-        assertThatThrownBy { profileService.getProfile("test-non-existing") }
+        assertThatThrownBy { profileService.getProfile(jobWithProfile("test-non-existing")) }
             .isInstanceOf(RuntimeException::class.java)
             .hasMessageStartingWith("Could not find location for profile test-non-existing! Profiles: {")
     }
 
     @Test
     fun `unreachable profile throws error`() {
-        assertThatThrownBy { profileService.getProfile("test-invalid-location") }
+        assertThatThrownBy { profileService.getProfile(jobWithProfile("test-invalid-location")) }
             .isInstanceOf(IOException::class.java)
             .hasMessage("class path resource [profile/test_profile_invalid_location.yml] cannot be opened because it does not exist")
     }
 
     @Test
     fun `unreachable profiles throws error`() {
-        profileService = ProfileService(ClassPathResource("nonexisting.yml"), objectMapper)
-        assertThatThrownBy { profileService.getProfile("test-profile") }
+        profileService = ProfileService(ProfileProperties(ClassPathResource("nonexisting.yml")), objectMapper)
+        assertThatThrownBy { profileService.getProfile(jobWithProfile("test-profile")) }
             .isInstanceOf(IOException::class.java)
             .hasMessage("class path resource [nonexisting.yml] cannot be opened because it does not exist")
     }
 
     @Test
     fun `profile value empty throw errrors`() {
-        assertThatThrownBy { profileService.getProfile("none") }
+        assertThatThrownBy { profileService.getProfile(jobWithProfile("none")) }
             .isInstanceOf(RuntimeException::class.java)
             .hasMessageStartingWith("Could not find location for profile none! Profiles: {")
     }
+
+    private fun jobWithProfile(profile: String) = defaultEncoreJob().copy(profile = profile)
 }
diff --git a/encore-common/src/test/resources/profile/archive.yml b/encore-common/src/test/resources/profile/archive.yml
@@ -3,11 +3,11 @@ description: Archive format
 encodes:
   - type: VideoEncode
     codec: dnxhd
-    height: 1080
+    height: #{profileParams['height']}
     params:
       b:v: 185M
       pix_fmt: yuv422p10le
-    suffix: _DNxHD_185x
+    suffix: #{profileParams['suffix']}
     format: mxf
     twoPass: false
     audioEncode:
