From e63c74813340f546cc2a6c0fb5215f6b840292b6 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 20:15:57 +0500 Subject: [PATCH 1/9] Rename pass MarkupClassNames to CalculateFullNamesAndSetSurroundingType The new name gives more information about what the pass does --- shared/src/main/scala/io/kaitai/struct/Main.scala | 2 +- ...cala => CalculateFullNamesAndSetSurroundingType.scala} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename shared/src/main/scala/io/kaitai/struct/precompile/{MarkupClassNames.scala => CalculateFullNamesAndSetSurroundingType.scala} (67%) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7ac34ef6e..7a03ba02b 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -39,7 +39,7 @@ object Main { * @return a list of compilation problems encountered during precompilation steps */ def precompile(specs: ClassSpecs, conf: RuntimeConfig): Iterable[CompilationProblem] = { - new MarkupClassNames(specs).run() + new CalculateFullNamesAndSetSurroundingType(specs).run() val resolveTypeProblems = specs.flatMap { case (_, topClass) => val config = updateConfigFromMeta(conf, topClass.meta) new ResolveTypes(specs, topClass, config.opaqueTypes).run() diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala similarity index 67% rename from shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala rename to shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala index 083aec982..798ed957e 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala @@ -3,13 +3,13 @@ package io.kaitai.struct.precompile import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.CompilationProblem -class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { +class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep { override def run(): Iterable[CompilationProblem] = { - classSpecs.foreach { case (_, curClass) => markupClassNames(curClass) } + classSpecs.foreach { case (_, curClass) => calculate(curClass) } None } - def markupClassNames(curClass: ClassSpec): Unit = { + private def calculate(curClass: ClassSpec): Unit = { curClass.enums.foreach { case (enumName, enumSpec) => enumSpec.name = curClass.name ::: List(enumName) } @@ -17,7 +17,7 @@ class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { curClass.types.foreach { case (nestedName: String, nestedClass) => nestedClass.name = curClass.name ::: List(nestedName) nestedClass.upClass = Some(curClass) - markupClassNames(nestedClass) + calculate(nestedClass) } } } From 90d440b5afd9c66fd2d6329b23dc1f423505e0f8 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 20:25:38 +0500 Subject: [PATCH 2/9] Document CalculateFullNamesAndSetSurroundingType pass and improve documentation of related ClassSpec and EnumSpec properties --- .../src/main/scala/io/kaitai/struct/format/ClassSpec.scala | 4 ++++ .../src/main/scala/io/kaitai/struct/format/EnumSpec.scala | 6 ++++++ .../CalculateFullNamesAndSetSurroundingType.scala | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala index 6a3cb0135..010516e00 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala @@ -58,6 +58,8 @@ case class ClassSpec( * Full absolute name of the class (including all names of classes that * it's nested into, as a namespace). Derived either from `meta`/`id` * (for top-level classes), or from keys in `types` (for nested classes). + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var name = List[String]() @@ -76,6 +78,8 @@ case class ClassSpec( /** * The class specification that this class is nested into, if it exists. * For top-level classes, it's None. + * + * This class is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var upClass: Option[ClassSpec] = None diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala index 6fcaf791e..5c41442f5 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala @@ -6,6 +6,12 @@ import scala.collection.immutable.SortedMap import scala.collection.mutable case class EnumSpec(path: List[String], map: SortedMap[Long, EnumValueSpec]) extends YAMLPath { + /** + * Absolute name of the enum (includes the names of all classes inside which + * it is defined). Derived either from keys in `enums`. + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. + */ var name = List[String]() /** diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala index 798ed957e..087a82f44 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala @@ -3,6 +3,10 @@ package io.kaitai.struct.precompile import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.CompilationProblem +/** + * Assigns a full name to the `.name` property to each KS type and enum. + * Also for each [[ClassSpec]] assigns [[ClassSpec.upClass]] to a surrounding type. + */ class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep { override def run(): Iterable[CompilationProblem] = { classSpecs.foreach { case (_, curClass) => calculate(curClass) } From 1b56b38b0deb0c000256dde6903ec8efa490d915 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 23:24:42 +0500 Subject: [PATCH 3/9] Rename pass SpecsValueTypeDerive to DeriveValueInstanceTypes The new name gives more information about what the pass does --- shared/src/main/scala/io/kaitai/struct/Main.scala | 2 +- ...csValueTypeDerive.scala => DeriveValueInstanceTypes.scala} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename shared/src/main/scala/io/kaitai/struct/precompile/{SpecsValueTypeDerive.scala => DeriveValueInstanceTypes.scala} (82%) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7a03ba02b..29978ff13 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -52,7 +52,7 @@ object Main { } new ParentTypes(specs).run() - new SpecsValueTypeDerive(specs).run() + new DeriveValueInstanceTypes(specs).run() new CalculateSeqSizes(specs).run() val typeValidatorProblems = new TypeValidator(specs).run() diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala similarity index 82% rename from shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala rename to shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index 74686d1a1..d88f076e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -3,13 +3,13 @@ package io.kaitai.struct.precompile import io.kaitai.struct.Log import io.kaitai.struct.format.ClassSpecs -class SpecsValueTypeDerive(specs: ClassSpecs) { +class DeriveValueInstanceTypes(specs: ClassSpecs) { def run(): Unit = { var iterNum = 1 var hasChanged = false do { hasChanged = false - Log.typeProcValue.info(() => s"### SpecsValueTypeDerive: iteration #$iterNum") + Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum") specs.foreach { case (specName, spec) => Log.typeProcValue.info(() => s"#### $specName") val thisChanged = new ValueTypesDeriver(specs, spec).run() From c23dd0fdf4c88b4ea4e4eac40c9fa036bd9d8006 Mon Sep 17 00:00:00 2001 From: Mingun Date: Fri, 15 Mar 2024 23:31:03 +0500 Subject: [PATCH 4/9] Document DeriveValueInstanceTypes pass and add documentation to related ValueInstanceSpec property --- .../main/scala/io/kaitai/struct/format/InstanceSpec.scala | 4 ++++ .../kaitai/struct/precompile/DeriveValueInstanceTypes.scala | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala index cbefec411..bbcfb8caf 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala @@ -13,6 +13,10 @@ case class ValueInstanceSpec( path: List[String], value: Ast.expr, ifExpr: Option[Ast.expr] = None, + /** + * Type of the `value`. Calculated by the [[DeriveValueInstanceTypes]] pass. + * `None` means "un-calculated yet". + */ var dataTypeOpt: Option[DataType] = None, val _doc: DocSpec = DocSpec.EMPTY, ) extends InstanceSpec(_doc) { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index d88f076e9..0d7992851 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -3,6 +3,12 @@ package io.kaitai.struct.precompile import io.kaitai.struct.Log import io.kaitai.struct.format.ClassSpecs +/** + * Assign types to value instances by deriving them from their expressions. + * + * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is + * a type of value instance. + */ class DeriveValueInstanceTypes(specs: ClassSpecs) { def run(): Unit = { var iterNum = 1 From 4233fcb1be60d52c207929160aa2d9905710db37 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 16:54:21 +0500 Subject: [PATCH 5/9] hasUndecided variable is not used, remove it --- .../scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala index e66c5c016..4dbea47ff 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala @@ -15,7 +15,6 @@ class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { def deriveValueType(curClass: ClassSpec): Boolean = { Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") var hasChanged = false - var hasUndecided = false provider.nowClass = curClass curClass.instances.foreach { @@ -32,7 +31,6 @@ class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { } catch { case tue: TypeUndecidedError => Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") - hasUndecided = true // just ignore, we're not there yet, probably we'll get it on next iteration case err: ExpressionError => throw ErrorInInput(err, vi.path ++ List("value")).toException From e0050d4aad44f561edbcc026872124a3c8927daa Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 16:46:22 +0500 Subject: [PATCH 6/9] Merge ValueTypesDetector into DeriveValueInstanceTypes because it is used only by that pass --- .../precompile/DeriveValueInstanceTypes.scala | 56 ++++++++++++++++++- .../struct/precompile/ValueTypesDeriver.scala | 54 ------------------ 2 files changed, 53 insertions(+), 57 deletions(-) delete mode 100644 shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index 0d7992851..e77d21cf0 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -1,7 +1,9 @@ package io.kaitai.struct.precompile -import io.kaitai.struct.Log -import io.kaitai.struct.format.ClassSpecs +import io.kaitai.struct.{ClassTypeProvider, Log} +import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} +import io.kaitai.struct.problems.ErrorInInput +import io.kaitai.struct.translators.TypeDetector /** * Assign types to value instances by deriving them from their expressions. @@ -18,7 +20,11 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum") specs.foreach { case (specName, spec) => Log.typeProcValue.info(() => s"#### $specName") - val thisChanged = new ValueTypesDeriver(specs, spec).run() + + val provider = new ClassTypeProvider(specs, spec) + val detector = new TypeDetector(provider) + + val thisChanged = deriveValueType(spec, provider, detector) Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes")) hasChanged |= thisChanged } @@ -26,4 +32,48 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { } while (hasChanged) Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") } + + private def deriveValueType( + curClass: ClassSpec, + provider: ClassTypeProvider, + detector: TypeDetector + ): Boolean = { + Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") + var hasChanged = false + + provider.nowClass = curClass; + curClass.instances.foreach { + case (instName, inst) => + inst match { + case vi: ValueInstanceSpec => + vi.dataTypeOpt match { + case None => + try { + val viType = detector.detectType(vi.value) + vi.dataTypeOpt = Some(viType) + Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") + hasChanged = true + } catch { + case tue: TypeUndecidedError => + Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") + // just ignore, we're not there yet, probably we'll get it on next iteration + case err: ExpressionError => + throw ErrorInInput(err, vi.path ++ List("value")).toException + } + case Some(_) => + // already derived, do nothing + } + case _ => + // do nothing + } + } + + // Continue with all nested types + curClass.types.foreach { + case (_, classSpec) => + hasChanged ||= deriveValueType(classSpec, provider, detector) + } + + hasChanged + } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala deleted file mode 100644 index 4dbea47ff..000000000 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ /dev/null @@ -1,54 +0,0 @@ -package io.kaitai.struct.precompile - -import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} -import io.kaitai.struct.problems.ErrorInInput -import io.kaitai.struct.translators.TypeDetector -import io.kaitai.struct.{ClassTypeProvider, Log} - -class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { - val provider = new ClassTypeProvider(specs, topClass) - val detector = new TypeDetector(provider) - - def run(): Boolean = - deriveValueType(topClass) - - def deriveValueType(curClass: ClassSpec): Boolean = { - Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") - var hasChanged = false - - provider.nowClass = curClass - curClass.instances.foreach { - case (instName, inst) => - inst match { - case vi: ValueInstanceSpec => - vi.dataTypeOpt match { - case None => - try { - val viType = detector.detectType(vi.value) - vi.dataTypeOpt = Some(viType) - Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") - hasChanged = true - } catch { - case tue: TypeUndecidedError => - Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") - // just ignore, we're not there yet, probably we'll get it on next iteration - case err: ExpressionError => - throw ErrorInInput(err, vi.path ++ List("value")).toException - } - case Some(_) => - // already derived, do nothing - } - case _ => - // do nothing - } - } - - // Continue with all nested types - curClass.types.foreach { - case (_, classSpec) => - hasChanged ||= deriveValueType(classSpec) - } - - hasChanged - } -} From bce949802169b3b4b3b3a37e60d4a14b0a78912a Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 17:41:19 +0500 Subject: [PATCH 7/9] Convert all passes to the PrecompileStep passes and run them all uniformly --- .../main/scala/io/kaitai/struct/Main.scala | 19 ++++++++++--------- .../struct/precompile/CalculateSeqSizes.scala | 6 ++++-- .../precompile/DeriveValueInstanceTypes.scala | 7 ++++--- .../struct/precompile/ParentTypes.scala | 6 ++++-- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 29978ff13..83bdaa854 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -51,16 +51,17 @@ object Main { return resolveTypeProblems } - new ParentTypes(specs).run() - new DeriveValueInstanceTypes(specs).run() - new CalculateSeqSizes(specs).run() - val typeValidatorProblems = new TypeValidator(specs).run() + var passes = Seq( + new ParentTypes(specs), + new DeriveValueInstanceTypes(specs), + new CalculateSeqSizes(specs), + new TypeValidator(specs), + // Warnings + new StyleCheckIds(specs), + new CanonicalizeEncodingNames(specs), + ) - // Warnings - val styleWarnings = new StyleCheckIds(specs).run() - val encodingProblems = new CanonicalizeEncodingNames(specs).run() - - resolveTypeProblems ++ typeValidatorProblems ++ styleWarnings ++ encodingProblems + resolveTypeProblems ++ passes.flatMap(_.run()) } /** diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 4d28c263b..688259fa0 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -5,10 +5,12 @@ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem -class CalculateSeqSizes(specs: ClassSpecs) { - def run(): Unit = { +class CalculateSeqSizes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { specs.forEachRec(CalculateSeqSizes.getSeqSize) + None } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index e77d21cf0..a8133c503 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -2,7 +2,7 @@ package io.kaitai.struct.precompile import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} -import io.kaitai.struct.problems.ErrorInInput +import io.kaitai.struct.problems.{CompilationProblem, ErrorInInput} import io.kaitai.struct.translators.TypeDetector /** @@ -11,8 +11,8 @@ import io.kaitai.struct.translators.TypeDetector * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is * a type of value instance. */ -class DeriveValueInstanceTypes(specs: ClassSpecs) { - def run(): Unit = { +class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { var iterNum = 1 var hasChanged = false do { @@ -31,6 +31,7 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) { iterNum += 1 } while (hasChanged) Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") + None } private def deriveValueType( diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala index 2929f4e5f..4a1389cd5 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala @@ -4,18 +4,20 @@ import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType.{ArrayTypeInStream, SwitchType, UserType} import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem import io.kaitai.struct.translators.TypeDetector /** * Precompile step that calculates actual parent types of KSY-defined types * (the type of the `_parent` built-in property). */ -class ParentTypes(classSpecs: ClassSpecs) { - def run(): Unit = { +class ParentTypes(classSpecs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { classSpecs.foreach { case (_, curClass) => markup(curClass) } classSpecs.forEachTopLevel((_, spec) => { spec.parentClass = GenericStructClassSpec }) + None } def markup(curClass: ClassSpec): Unit = { From b5a9c313f3a89bb21ed61814e8f061cc378e4267 Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 18:33:43 +0500 Subject: [PATCH 8/9] Allow to run warning passes on model with erroneous value instances Currently the pass DeriveValueInstanceTypes throws error if type of instance cannot be calculated, but this will change in next commit. That means that accessing `dataType` of the ValueInstanceSpec will throw an error. If such error is thrown we just cannot analyze that instance and can just ignore it (review this commit in whitespace changes ignored mode) --- .../CanonicalizeEncodingNames.scala | 27 ++++++++------ .../struct/precompile/StyleCheckIds.scala | 35 +++++++++++-------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala index 6091311d8..1904c8e95 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala @@ -32,16 +32,23 @@ object CanonicalizeEncodingNames { } def canonicalizeMember(member: MemberSpec): Iterable[CompilationProblem] = { - (member.dataType match { - case strType: StrFromBytesType => - val (newEncoding, problem1) = canonicalizeName(strType.encoding) - strType.encoding = newEncoding - // Do not report problem if encoding was derived from `meta/encoding` key - if (strType.isEncodingDerived) None else problem1 - case _ => - // not a string type = no problem - None - }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + try { + (member.dataType match { + case strType: StrFromBytesType => + val (newEncoding, problem1) = canonicalizeName(strType.encoding) + strType.encoding = newEncoding + // Do not report problem if encoding was derived from `meta/encoding` key + if (strType.isEncodingDerived) None else problem1 + case _ => + // not a string type = no problem + None + }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def canonicalizeName(original: String): (String, Option[CompilationProblem with PathLocalizable]) = { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala index 4c4753847..597f3d9b4 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala @@ -40,20 +40,27 @@ class StyleCheckIds(specs: ClassSpecs) extends PrecompileStep { } def getSizeRefProblem(spec: ClassSpec, attr: MemberSpec): Option[CompilationProblem] = { - getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { - val existingName = sizeRefAttr.id.humanReadable - val goodName = s"len_${attr.id.humanReadable}" - if (existingName != goodName) { - Some(StyleWarningSizeLen( - goodName, - existingName, - attr.id.humanReadable, - ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) - )) - } else { - None - } - }) + try { + getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { + val existingName = sizeRefAttr.id.humanReadable + val goodName = s"len_${attr.id.humanReadable}" + if (existingName != goodName) { + Some(StyleWarningSizeLen( + goodName, + existingName, + attr.id.humanReadable, + ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) + )) + } else { + None + } + }) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def getRepeatExprRefProblem(spec: ClassSpec, attr: AttrLikeSpec): Option[CompilationProblem] = { From 6aecb1e58a04c9b4662f4a8713ff4798ed82884a Mon Sep 17 00:00:00 2001 From: Mingun Date: Sat, 5 Oct 2024 22:10:07 +0500 Subject: [PATCH 9/9] Ignore all errors in DeriveValueInstanceTypes, errors will be detected by `TypeValidator` If just return CompilationProblems from this pass, we will report some errors twice, because `TypeValidator` also reports errors from the `TypeDetector` --- .../kaitai/struct/precompile/DeriveValueInstanceTypes.scala | 2 +- .../scala/io/kaitai/struct/precompile/TypeValidator.scala | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala index a8133c503..de42d4b4b 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -59,7 +59,7 @@ class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep { Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") // just ignore, we're not there yet, probably we'll get it on next iteration case err: ExpressionError => - throw ErrorInInput(err, vi.path ++ List("value")).toException + // Ignore all errors, the validation will be performed in TypeValidator pass later } case Some(_) => // already derived, do nothing diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala index e3e1b6dd1..d9a0b3665 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala @@ -105,6 +105,10 @@ class TypeValidator(specs: ClassSpecs) extends PrecompileStep { def validateValueInstance(vis: ValueInstanceSpec): Option[CompilationProblem] = { try { + // detectType performs some additional checks that validate does not (for example, + // applicability of operators to types, like `Not` to numbers). + // TODO: probably implement those checks in validate too? + detector.detectType(vis.value) detector.validate(vis.value) None } catch {