Merge pull request #3738 from rbayet/feat_analysis_tweaks

[Core] Analysis: introducing constant and token filters for special characters handling

Author: rbayet

src/module-elasticsuite-core/Api/Index/Mapping/FieldInterface.php

@@ -52,6 +52,17 @@ interface FieldInterface
     const ANALYZER_REFERENCE  = 'reference';
     const ANALYZER_EDGE_NGRAM = 'standard_edge_ngram';
 
+    /**
+     * Token filter types declarations.
+     */
+    const TOKEN_FILTER_TYPE_WORD_DELIMITER = 'word_delimiter_graph';
+    const TOKEN_FILTER_TYPE_REMOVE_DUPLICATES = 'remove_duplicates';
+
+    /**
+     * Tokenizer declarations.
+     */
+    const TOKENIZER_WHITESPACE = 'whitespace';
+
     /**
      * Field filter logical operators.
      */

src/module-elasticsuite-core/etc/elasticsuite_analysis.xml

@@ -29,6 +29,7 @@
             <length>8192</length>
         </filter>
         <filter name="lowercase" type="lowercase" language="default"/>
+        <filter name="word_delimiter_before" type="trim" language="default" />
         <filter name="word_delimiter" type="word_delimiter_graph" language="default">
             <generate_word_parts>true</generate_word_parts>
             <generate_number_parts>true</generate_number_parts>
@@ -39,6 +40,7 @@
             <split_on_numerics>true</split_on_numerics>
             <preserve_original>true</preserve_original>
         </filter>
+        <filter name="remove_duplicates" type="remove_duplicates" language="default" />
         <filter name="shingle" type="shingle" language="default">
             <min_shingle_size>2</min_shingle_size>
             <max_shingle_size>4</max_shingle_size>
@@ -216,6 +218,7 @@
                 <filter ref="ascii_folding" />
                 <filter ref="trim" />
                 <filter ref="elision" />
+                <filter ref="word_delimiter_before" />
                 <filter ref="word_delimiter" />
                 <filter ref="lowercase" />
                 <filter ref="stemmer_before" />
@@ -232,6 +235,7 @@
                 <filter ref="ascii_folding" />
                 <filter ref="trim" />
                 <filter ref="elision" />
+                <filter ref="word_delimiter_before" />
                 <filter ref="word_delimiter" />
                 <filter ref="lowercase" />
             </filters>
@@ -260,6 +264,7 @@
                 <filter ref="ascii_folding" />
                 <filter ref="trim" />
                 <filter ref="elision" />
+                <filter ref="word_delimiter_before" />
                 <filter ref="word_delimiter" />
                 <filter ref="lowercase" />
                 <filter ref="stemmer_before" />
@@ -300,6 +305,7 @@
                 <filter ref="ascii_folding" />
                 <filter ref="trim" />
                 <filter ref="elision" />
+                <filter ref="word_delimiter_before" />
                 <filter ref="word_delimiter" />
                 <filter ref="lowercase" />
                 <filter ref="stemmer_before" />