Add support for annotations for Columns and ColSpecs (#1190)

* Add support for stereotypes and tagged values to Columns/ColSpecs

* fix: Fix setting stereotypesCoreInstance

* feat: Add colSpec function that takes Column param

* feat: Add new function signature to FunctionExpressionProcessor

* feat: Add TestColSpecAnnotations

* feat: Call new colSpec method if column has stereotypes/tagged values

* feat: Add tests for col spec

* feat: Add test to TestColSpecAnnotations

* feat: Try not overloading the colSpec function

* refactor: Make separate generateStereotypesBuilder and generateTaggedValuesBuilder functions

* feat: Update TestColSpecAnnotations

* fix: Fix processing tagged values in _Column

* feat: Add tests to TestColSpecAnnotations

* feat: Add overloaded getColumnInstance function that takes string type

* feat: Remove extends AnnotatedElement from ColSpec class

* fix: Remove unused import

* feat: Feedback from comments

* refactor: Update to getUserPathForPackageableElement

* fix: Resolve import stubs in _RelationType

* feat: Update getColumnInstance functions to handle resolved import stubs

* fix: Fix paths in GenericTypeSerializationInCode

* fix: Set assumeNoImportStubs flag to true

* fix: Uncomment tests

* fix: Update GenericType functions to pass stereotypes and tagged values

* fix: Propagate stereotypes/tagged values wherever _Column.getColumnInstance is used

* fix: Ensure correct passing of _stereotypes()/_stereotypesCoreInstance()

* refactor: Re-add overloaded function calls

* refactor: Add docs to getColumnInstance

* fix: Add missing package

* fix: Change to using _stereotypes() in GenericType

* fix: Remove stereotypes/taggeed value passing in TDSExtension

* refactor: Make separate function names for compiled vs. interpreted

* Revert "refactor: Make separate function names for compiled vs. interpreted"

This reverts commit 05b4cae.

* refactor: Remove assumeNoImportStubs param

* fix: Merge stereotypes/tagged values in merge function

* fix: Ensure only distinct stereotypes/tagged values

* fix: Use _stereotypes instead of _stereotypesCoreInstance in merge function

Author: travisstebbins

legend-pure-core/legend-pure-m3-core/src/main/antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4

@@ -315,7 +315,7 @@ atomicExpression:
 
 columnBuilders: TILDE (oneColSpec | (BRACKET_OPEN (oneColSpec(COMMA oneColSpec)*)? BRACKET_CLOSE))
 ;
-oneColSpec: columnName (COLON (type multiplicity? | anyLambda) extraFunction?)?
+oneColSpec: stereotypes? taggedValues? columnName (COLON (type multiplicity? | anyLambda) extraFunction?)?
 ;
 extraFunction: (COLON anyLambda)
 ;

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/compiler/postprocessing/processor/valuespecification/FunctionExpressionProcessor.java

@@ -407,7 +407,7 @@ private static boolean manageMagicColumnFunctions(FunctionExpression functionExp
             }
             else
             {
-                ctx.register((GenericType) processorSupport.function_getFunctionType(foundFunction).getValueForMetaPropertyToMany("parameters").get(1).getValueForMetaPropertyToOne("genericType"), (GenericType) processorSupport.type_wrapGenericType(_RelationType.build(found.collect(foundC -> _Column.getColumnInstance(foundC._name(), false, _Column.getColumnType(foundC), _Column.getColumnMultiplicity(foundC), functionExpression.getSourceInformation(), processorSupport)), functionExpression.getSourceInformation(), processorSupport)), ctx, observer);
+                ctx.register((GenericType) processorSupport.function_getFunctionType(foundFunction).getValueForMetaPropertyToMany("parameters").get(1).getValueForMetaPropertyToOne("genericType"), (GenericType) processorSupport.type_wrapGenericType(_RelationType.build(found.collect(foundC -> _Column.getColumnInstance(foundC._name(), false, _Column.getColumnType(foundC), _Column.getColumnMultiplicity(foundC), foundC._stereotypesCoreInstance(), foundC._taggedValues(), functionExpression.getSourceInformation(), processorSupport)), functionExpression.getSourceInformation(), processorSupport)), ctx, observer);
                 columnTypeInferenceSuccess = true;
             }
         }
@@ -481,7 +481,7 @@ private static boolean manageMagicColumnFunctions(FunctionExpression functionExp
             }
             else
             {
-                ctx.register((GenericType) processorSupport.function_getFunctionType(foundFunction).getValueForMetaPropertyToMany("parameters").get(1).getValueForMetaPropertyToOne("genericType"), (GenericType) processorSupport.type_wrapGenericType(_RelationType.build(found.collect(foundC -> _Column.getColumnInstance(foundC._name(), false, _Column.getColumnType(foundC), _Column.getColumnMultiplicity(foundC), functionExpression.getSourceInformation(), processorSupport)), functionExpression.getSourceInformation(), processorSupport)), ctx, observer);
+                ctx.register((GenericType) processorSupport.function_getFunctionType(foundFunction).getValueForMetaPropertyToMany("parameters").get(1).getValueForMetaPropertyToOne("genericType"), (GenericType) processorSupport.type_wrapGenericType(_RelationType.build(found.collect(foundC -> _Column.getColumnInstance(foundC._name(), false, _Column.getColumnType(foundC), _Column.getColumnMultiplicity(foundC), foundC._stereotypesCoreInstance(), foundC._taggedValues(), functionExpression.getSourceInformation(), processorSupport)), functionExpression.getSourceInformation(), processorSupport)), ctx, observer);
                 columnTypeInferenceSuccess = true;
             }
         }

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/generictype/GenericType.java

@@ -153,6 +153,8 @@ public static CoreInstance makeTypeArgumentAsConcreteAsPossible(CoreInstance typ
                                     c._nameWildCard(),
                                     (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType) makeTypeArgumentAsConcreteAsPossible(_Column.getColumnType(c), filteredGenericTypeByTypeParameterNames, sourceMulBinding, processorSupport),
                                     _Column.getColumnMultiplicity(c),
+                                    c._stereotypes(),
+                                    c._taggedValues(),
                                     c.getSourceInformation(),
                                     processorSupport
                             ),
@@ -283,7 +285,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.ge
 
         // Set RelationType on Generic
         return ((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType) processorSupport.newGenericType(null, operation, true))
-                ._rawType(_RelationType.build(newColumnSet.collect(c -> _Column.getColumnInstance(c._name(), c._nameWildCard(), _Column.getColumnType(c), _Column.getColumnMultiplicity(c), c.getSourceInformation(), processorSupport)), gLeft._rawType().getSourceInformation(), processorSupport));
+                ._rawType(_RelationType.build(newColumnSet.collect(c -> _Column.getColumnInstance(c._name(), c._nameWildCard(), _Column.getColumnType(c), _Column.getColumnMultiplicity(c), c._stereotypes(), c._taggedValues(), c.getSourceInformation(), processorSupport)), gLeft._rawType().getSourceInformation(), processorSupport));
     }
 
     @Deprecated
@@ -1313,6 +1315,8 @@ else if (_RelationType.isRelationType(type, processorSupport))
                                     c._nameWildCard(),
                                     (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType) copyGenericType(_Column.getColumnType(c), replaceSourceInfo, newSourceInfo, processorSupport, inferred),
                                     _Column.getColumnMultiplicity(c),
+                                    c._stereotypes(),
+                                    c._taggedValues(),
                                     replaceSourceInfo ? newSourceInfo : src.getSourceInformation(),
                                     processorSupport
                             ),

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/relation/_Column.java

@@ -14,8 +14,10 @@
 
 package org.finos.legend.pure.m3.navigation.relation;
 
-import java.util.Objects;
+import org.eclipse.collections.api.RichIterable;
 import org.eclipse.collections.api.factory.Lists;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.extension.Stereotype;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.extension.TaggedValue;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Column;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type;
@@ -34,6 +36,16 @@
 public class _Column
 {
     public static Column<?, ?> getColumnInstance(String name, boolean nameWildCard, String type, Multiplicity multiplicity, SourceInformation src, ProcessorSupport processorSupport)
+    {
+        return getColumnInstance(name, nameWildCard, type, multiplicity, null, null, src, processorSupport);
+    }
+
+    public static Column<?, ?> getColumnInstance(String name, boolean nameWildCard, GenericType targetType, Multiplicity multiplicity, SourceInformation sourceInformation, ProcessorSupport processorSupport)
+    {
+        return getColumnInstance(name, nameWildCard, targetType, multiplicity, null, null, sourceInformation, processorSupport);
+    }
+
+    public static Column<?, ?> getColumnInstance(String name, boolean nameWildCard, String type, Multiplicity multiplicity, RichIterable<? extends CoreInstance> stereotypes, RichIterable<? extends TaggedValue> taggedValues, SourceInformation src, ProcessorSupport processorSupport)
     {
         GenericType target = (GenericType) processorSupport.newAnonymousCoreInstance(src, M3Paths.GenericType);
         if (type == null)
@@ -49,10 +61,22 @@ public class _Column
             }
             target._rawType((Type)_type);
         }
-        return _Column.getColumnInstance(name, nameWildCard, target, multiplicity, src, processorSupport);
+        return _Column.getColumnInstance(name, nameWildCard, target, multiplicity, stereotypes, taggedValues, src, processorSupport);
     }
 
-    public static Column<?, ?> getColumnInstance(String name, boolean nameWildCard, GenericType targetType, org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity multiplicity, SourceInformation sourceInformation, ProcessorSupport processorSupport)
+    /**
+     * Create a Column instance with the given parameters. The targetType is expected to be a GenericType with the raw type of Column and one type argument for the column type. The multiplicity is expected to be the multiplicity argument for the column generic type.
+     * @param name the column name, which may be a wildcard (e.g. "?") if nameWildCard is true. If the name is quoted, the quotes will be removed.
+     * @param nameWildCard indicates whether the name is a wildcard. If true, the name will be treated as a wildcard and the column will match any name. If false, the name will be treated as a literal name.
+     * @param targetType the generic type for the column.
+     * @param multiplicity the multiplicity for the column.
+     * @param stereotypes the stereotypes to apply to the column.
+     * @param taggedValues the tagged values to apply to the column.
+     * @param sourceInformation the source information for the column instance, used for error reporting and debugging.
+     * @param processorSupport the processor support used for creating the column instance.
+     * @return a Column instance with the specified properties and metadata.
+     */
+    public static Column<?, ?> getColumnInstance(String name, boolean nameWildCard, GenericType targetType, Multiplicity multiplicity, RichIterable<? extends CoreInstance> stereotypes, RichIterable<? extends TaggedValue> taggedValues, SourceInformation sourceInformation, ProcessorSupport processorSupport)
     {
         Column<?, ?> columnInstance = (Column<?, ?>) processorSupport.newAnonymousCoreInstance(sourceInformation, M3Paths.Column);
         columnInstance._name(StringEscape.unescape(removeQuotes(name)));
@@ -63,6 +87,21 @@ public class _Column
         columnGenericType._typeArguments(Lists.mutable.with(null, targetType));
         columnGenericType._multiplicityArgumentsAdd(multiplicity);
         columnInstance.setKeyValues(M3PropertyPaths.classifierGenericType, Lists.mutable.with(columnGenericType));
+        if (stereotypes != null && stereotypes.notEmpty())
+        {
+            if (stereotypes.allSatisfy(s -> s instanceof Stereotype))
+            {
+                columnInstance._stereotypes(ListHelper.wrapListIterable(stereotypes).selectInstancesOf(Stereotype.class));
+            }
+            else
+            {
+                columnInstance._stereotypesCoreInstance(stereotypes);
+            }
+        }
+        if (taggedValues != null && taggedValues.notEmpty())
+        {
+            columnInstance._taggedValues(taggedValues);
+        }
         return columnInstance;
     }
 

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/navigation/relation/_RelationType.java

@@ -24,6 +24,8 @@
 import org.eclipse.collections.api.set.MutableSet;
 import org.eclipse.collections.api.tuple.Pair;
 import org.eclipse.collections.impl.tuple.Tuples;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.extension.Stereotype;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.extension.TaggedValue;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.Function;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionAccessor;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Column;
@@ -225,7 +227,13 @@ public static GenericType merge(GenericType existingGenericType, GenericType gen
                             Type rawTypeB = (Type) Instance.getValueForMetaPropertyToOneResolved(b, M3Properties.rawType, processorSupport);
                             GenericType merged = rawTypeA == null ? b : rawTypeB == null ? a : (GenericType) org.finos.legend.pure.m3.navigation.generictype.GenericType.findBestCommonGenericType(Lists.mutable.with(a, b), isCovariant, false, genericTypeCopy.getSourceInformation(), processorSupport);
                             org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity mergedMul = rawTypeA == null ? _Column.getColumnMultiplicity(c.getTwo()) : rawTypeB == null ? _Column.getColumnMultiplicity(c.getOne()) : (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity) Multiplicity.minSubsumingMultiplicity(_Column.getColumnMultiplicity(c.getOne()), _Column.getColumnMultiplicity(c.getTwo()), processorSupport);
-                            return _Column.getColumnInstance(cName, wildcard, merged, mergedMul, null, processorSupport);
+                            RichIterable<? extends Stereotype> stereotypesA = c.getOne()._stereotypes() == null ? Lists.mutable.empty() : c.getOne()._stereotypes();
+                            RichIterable<? extends Stereotype> stereotypesB = c.getTwo()._stereotypes() == null ? Lists.mutable.empty() : c.getTwo()._stereotypes();
+                            RichIterable<? extends Stereotype> mergedStereotypes = Lists.mutable.<Stereotype>withAll(stereotypesA).withAll(stereotypesB).distinct();
+                            RichIterable<? extends TaggedValue> taggedValuesA = c.getOne()._taggedValues() == null ? Lists.mutable.empty() : c.getOne()._taggedValues();
+                            RichIterable<? extends TaggedValue> taggedValuesB = c.getTwo()._taggedValues() == null ? Lists.mutable.empty() : c.getTwo()._taggedValues();
+                            RichIterable<? extends TaggedValue> mergedTaggedValues = Lists.mutable.<TaggedValue>withAll(taggedValuesA).withAll(taggedValuesB).distinctBy(TaggedValue::_tagCoreInstance);
+                            return _Column.getColumnInstance(cName, wildcard, merged, mergedMul, mergedStereotypes, mergedTaggedValues, null, processorSupport);
                         }),
                         existingGenericType.getValueForMetaPropertyToOne(M3Properties.rawType).getSourceInformation(),
                         processorSupport
@@ -256,6 +264,8 @@ public static void resolveImportStubs(CoreInstance relationType, ProcessorSuppor
     {
         relationType.getValueForMetaPropertyToMany(M3Properties.columns).forEach(c ->
         {
+            c.getValueForMetaPropertyToMany(M3Properties.stereotypes).forEach(s -> ImportStub.withImportStubByPass(s, processorSupport));
+            c.getValueForMetaPropertyToMany(M3Properties.taggedValues).forEach(tv -> ImportStub.withImportStubByPass(tv.getValueForMetaPropertyToOne(M3Properties.tag), processorSupport));
             CoreInstance classifierGenericType = c.getValueForMetaPropertyToOne(M3Properties.classifierGenericType);
             classifierGenericType.getValueForMetaPropertyToMany(M3Properties.multiplicityArguments).forEach(arg -> ImportStub.withImportStubByPass(arg, processorSupport));
             // the first type argument of the column type is the relation type itself: we skip it to avoid infinite recursion

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java

@@ -77,6 +77,7 @@
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.Multiplicity;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.MultiplicityInstance;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.multiplicity.MultiplicityValueInstance;
+import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.Column;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.GenericTypeOperationInstance;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relationship.Association;
@@ -950,6 +951,11 @@ else if (ctx.columnBuilders() != null)
                 columnNames.add(this.repository.newStringCoreInstance(colName));
                 GenericType returnType = null;
                 Multiplicity multiplicity = (Multiplicity) org.finos.legend.pure.m3.navigation.multiplicity.Multiplicity.newMultiplicity(0, 1, processorSupport);
+
+                // Extract stereotypes and tagged values
+                RichIterable<? extends CoreInstance> stereotypes = (oneColSpec.stereotypes() == null) ? null : stereotypes(oneColSpec.stereotypes(), importId);
+                RichIterable<? extends TaggedValue> taggedValues = (oneColSpec.taggedValues() == null) ? null : taggedValues(oneColSpec.taggedValues(), importId);
+
                 if (oneColSpec.anyLambda() != null)
                 {
                     lambdas.add(processLambda(oneColSpec.anyLambda(), Lists.mutable.empty(), Lists.mutable.empty(), lambdaContext, importId, space, addLines, false, Lists.mutable.empty()));
@@ -972,7 +978,7 @@ else if (oneColSpec.type() != null)
                     returnType = (GenericType) processorSupport.newAnonymousCoreInstance(src, M3Paths.GenericType);
                     returnType._rawType(null);
                 }
-                columnInstances.add(_Column.getColumnInstance(colName, false, returnType, multiplicity, src, processorSupport));
+                columnInstances.add(_Column.getColumnInstance(colName, false, returnType, multiplicity, stereotypes, taggedValues, src, processorSupport));
             });
             RelationType<?> relationType = _RelationType.build(columnInstances, this.sourceInformation.getPureSourceInformation(ctx.getStart(), ctx.getStart(), ctx.getStop()), processorSupport);
             GenericType relationTypeGenericType = (GenericType) processorSupport.type_wrapGenericType(relationType);