Skip to content

Commit e241782

Browse files
felipeeliafelipeelia
authored
Merge pull request #4263 from 10up/fix/4212
Introduce new ep_asciifolding filter
2 parents 09c075a + 15a7e68 commit e241782

9 files changed

Lines changed: 149 additions & 24 deletions

File tree

‎includes/mappings/comment/7-0.php‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,14 @@
6565
'default' => [
6666
'tokenizer' => 'standard',
6767
/* This filter is documented in includes/mappings/post/7-0.php */
68-
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
68+
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
6969
/* This filter is documented in includes/mappings/post/7-0.php */
7070
'language' => apply_filters( 'ep_analyzer_language', 'english', 'analyzer_default' ),
7171
],
7272
'default_search' => [
7373
'tokenizer' => 'standard',
7474
/* This filter is documented in includes/mappings/post/7-0.php */
75-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
75+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
7676
/* This filter is documented in includes/mappings/post/7-0.php */
7777
'char_filter' => apply_filters( 'ep_default_search_analyzer_char_filters', [ 'html_strip' ] ),
7878
/* This filter is documented above */
@@ -90,27 +90,31 @@
9090
],
9191
],
9292
'filter' => [
93-
'shingle_filter' => [
93+
'shingle_filter' => [
9494
'type' => 'shingle',
9595
'min_shingle_size' => 2,
9696
'max_shingle_size' => 5,
9797
],
98-
'ewp_snowball' => [
98+
'ewp_snowball' => [
9999
'type' => 'snowball',
100100
/* This filter is documented in includes/mappings/post/7-0.php */
101101
'language' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ewp_snowball' ),
102102
],
103-
'edge_ngram' => [
103+
'edge_ngram' => [
104104
'max_gram' => 10,
105105
'min_gram' => 3,
106106
'type' => 'edge_ngram',
107107
],
108-
'ep_stop' => [
108+
'ep_stop' => [
109109
'type' => 'stop',
110110
'ignore_case' => true,
111111
/* This filter is documented in includes/mappings/post/7-0.php */
112112
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
113113
],
114+
'ep_asciifolding' => [
115+
'type' => 'asciifolding',
116+
'preserve_original' => true,
117+
],
114118
],
115119
'normalizer' => [
116120
'lowerasciinormalizer' => [

‎includes/mappings/comment/initial.php‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,14 @@
4949
'default' => [
5050
'tokenizer' => 'standard',
5151
/* This filter is documented in includes/mappings/post/7-0.php */
52-
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
52+
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
5353
/* This filter is documented in includes/mappings/post/7-0.php */
5454
'language' => apply_filters( 'ep_analyzer_language', 'english', 'analyzer_default' ),
5555
],
5656
'default_search' => [
5757
'tokenizer' => 'standard',
5858
/* This filter is documented in includes/mappings/post/7-0.php */
59-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
59+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
6060
/* This filter is documented in includes/mappings/post/7-0.php */
6161
'char_filter' => apply_filters( 'ep_default_search_analyzer_char_filters', [ 'html_strip' ] ),
6262
/* This filter is documented above */
@@ -100,6 +100,10 @@
100100
/* This filter is documented in includes/mappings/post/7-0.php */
101101
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
102102
],
103+
'ep_asciifolding' => [
104+
'type' => 'asciifolding',
105+
'preserve_original' => true,
106+
],
103107
],
104108
'normalizer' => [
105109
'lowerasciinormalizer' => [

‎includes/mappings/post/5-2.php‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
'default' => array(
5050
'tokenizer' => 'standard',
5151
/* This filter is documented in includes/mappings/post/7-0.php */
52-
'filter' => apply_filters( 'ep_default_analyzer_filters', array( 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball' ) ),
52+
'filter' => apply_filters( 'ep_default_analyzer_filters', array( 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ) ),
5353
/* This filter is documented in includes/mappings/post/7-0.php */
5454
'char_filter' => apply_filters( 'ep_default_analyzer_char_filters', array( 'html_strip' ) ),
5555
/* This filter is documented in includes/mappings/post/7-0.php */
@@ -58,7 +58,7 @@
5858
'default_search' => array(
5959
'tokenizer' => 'standard',
6060
/* This filter is documented in includes/mappings/post/7-0.php */
61-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', array( 'standard', 'lowercase', 'ep_stop', 'ewp_snowball' ) ),
61+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', array( 'standard', 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ) ),
6262
/* This filter is documented in includes/mappings/post/7-0.php */
6363
'char_filter' => apply_filters( 'ep_default_search_analyzer_char_filters', array( 'html_strip' ) ),
6464
/* This filter is documented in includes/mappings/post/7-0.php */
@@ -102,6 +102,10 @@
102102
/* This filter is documented in includes/mappings/post/7-0.php */
103103
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
104104
],
105+
'ep_asciifolding' => [
106+
'type' => 'asciifolding',
107+
'preserve_original' => true,
108+
],
105109
),
106110
'normalizer' => array(
107111
'lowerasciinormalizer' => array(

‎includes/mappings/post/7-0.php‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
* @param {array<string>} $filters Default filters
7373
* @return {array<string>} New filters
7474
*/
75-
'filter' => apply_filters( 'ep_default_analyzer_filters', array( 'lowercase', 'ep_stop', 'ewp_snowball' ) ),
75+
'filter' => apply_filters( 'ep_default_analyzer_filters', array( 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ) ),
7676
/**
7777
* Filter Elasticsearch default analyzer's char_filter
7878
*
@@ -102,7 +102,7 @@
102102
* @param {array<string>} $filters Default filters
103103
* @return {array<string>} New filters
104104
*/
105-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', array( 'lowercase', 'ep_stop', 'ewp_snowball' ) ),
105+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', array( 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ) ),
106106
/**
107107
* Filter Elasticsearch default analyzer's char_filter
108108
*
@@ -127,27 +127,31 @@
127127
),
128128
),
129129
'filter' => array(
130-
'shingle_filter' => array(
130+
'shingle_filter' => array(
131131
'type' => 'shingle',
132132
'min_shingle_size' => 2,
133133
'max_shingle_size' => 5,
134134
),
135-
'ewp_snowball' => array(
135+
'ewp_snowball' => array(
136136
'type' => 'snowball',
137137
/* This filter is documented in includes/mappings/post/7-0.php */
138138
'language' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ewp_snowball' ),
139139
),
140-
'edge_ngram' => array(
140+
'edge_ngram' => array(
141141
'max_gram' => 10,
142142
'min_gram' => 3,
143143
'type' => 'edge_ngram',
144144
),
145-
'ep_stop' => [
145+
'ep_stop' => [
146146
'type' => 'stop',
147147
'ignore_case' => true,
148148
/* This filter is documented in includes/mappings/post/7-0.php */
149149
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
150150
],
151+
'ep_asciifolding' => [
152+
'type' => 'asciifolding',
153+
'preserve_original' => true,
154+
],
151155
),
152156
'normalizer' => array(
153157
'lowerasciinormalizer' => array(

‎includes/mappings/term/7-0.php‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@
2929
'default' => [
3030
'tokenizer' => 'standard',
3131
/* This filter is documented in includes/mappings/post/7-0.php */
32-
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
32+
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
3333
/* This filter is documented in includes/mappings/post/7-0.php */
3434
'language' => apply_filters( 'ep_analyzer_language', 'english', 'analyzer_default' ),
3535
],
3636
'default_search' => [
3737
'tokenizer' => 'standard',
3838
/* This filter is documented in includes/mappings/post/7-0.php */
39-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
39+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
4040
/* This filter is documented in includes/mappings/post/7-0.php */
4141
'char_filter' => apply_filters( 'ep_default_search_analyzer_char_filters', [ 'html_strip' ] ),
4242
/* This filter is documented above */
@@ -54,27 +54,31 @@
5454
],
5555
],
5656
'filter' => [
57-
'shingle_filter' => [
57+
'shingle_filter' => [
5858
'type' => 'shingle',
5959
'min_shingle_size' => 2,
6060
'max_shingle_size' => 5,
6161
],
62-
'ewp_snowball' => [
62+
'ewp_snowball' => [
6363
'type' => 'snowball',
6464
/* This filter is documented in includes/mappings/post/7-0.php */
6565
'language' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ewp_snowball' ),
6666
],
67-
'edge_ngram' => [
67+
'edge_ngram' => [
6868
'max_gram' => 10,
6969
'min_gram' => 3,
7070
'type' => 'edge_ngram',
7171
],
72-
'ep_stop' => [
72+
'ep_stop' => [
7373
'type' => 'stop',
7474
'ignore_case' => true,
7575
/* This filter is documented in includes/mappings/post/7-0.php */
7676
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
7777
],
78+
'ep_asciifolding' => [
79+
'type' => 'asciifolding',
80+
'preserve_original' => true,
81+
],
7882
],
7983
'normalizer' => [
8084
'lowerasciinormalizer' => [

‎includes/mappings/term/initial.php‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@
1919
'default' => [
2020
'tokenizer' => 'standard',
2121
/* This filter is documented in includes/mappings/post/7-0.php */
22-
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
22+
'filter' => apply_filters( 'ep_default_analyzer_filters', [ 'standard', 'ewp_word_delimiter', 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
2323
'language' => apply_filters( 'ep_analyzer_language', 'english', 'analyzer_default' ),
2424
],
2525
'default_search' => [
2626
'tokenizer' => 'standard',
2727
/* This filter is documented in includes/mappings/post/7-0.php */
28-
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball' ] ),
28+
'filter' => apply_filters( 'ep_default_search_analyzer_filters', [ 'lowercase', 'ep_stop', 'ewp_snowball', 'ep_asciifolding' ] ),
2929
/* This filter is documented in includes/mappings/post/7-0.php */
3030
'char_filter' => apply_filters( 'ep_default_search_analyzer_char_filters', [ 'html_strip' ] ),
3131
/* This filter is documented above */
@@ -69,6 +69,10 @@
6969
/* This filter is documented in includes/mappings/post/7-0.php */
7070
'stopwords' => apply_filters( 'ep_analyzer_language', 'english', 'filter_ep_stop' ),
7171
],
72+
'ep_asciifolding' => [
73+
'type' => 'asciifolding',
74+
'preserve_original' => true,
75+
],
7276
],
7377
'normalizer' => [
7478
'lowerasciinormalizer' => [

‎tests/php/indexables/TestComment.php‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2650,6 +2650,22 @@ public function test_mapping_ep_stop_filter() {
26502650
$this->assertSame( '_arabic_', $index_settings['index.analysis.filter.ep_stop.stopwords'] );
26512651
}
26522652

2653+
/**
2654+
* Test the `ep_asciifolding` filter
2655+
*
2656+
* @since 5.3.3
2657+
* @group comment
2658+
*/
2659+
public function test_mapping_ep_asciifolding_filter() {
2660+
$indexable = ElasticPress\Indexables::factory()->get( 'comment' );
2661+
$index_name = $indexable->get_index_name();
2662+
$settings = ElasticPress\Elasticsearch::factory()->get_index_settings( $index_name );
2663+
$index_settings = $settings[ $index_name ]['settings'];
2664+
2665+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default.filter'] );
2666+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default_search.filter'] );
2667+
}
2668+
26532669
/**
26542670
* Test comment query with orderby none.
26552671
*

‎tests/php/indexables/TestPost.php‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9736,6 +9736,75 @@ public function test_mapping_ep_stop_filter() {
97369736
$this->assertSame( '_arabic_', $index_settings['index.analysis.filter.ep_stop.stopwords'] );
97379737
}
97389738

9739+
/**
9740+
* Test the `ep_asciifolding` filter
9741+
*
9742+
* @since 5.3.3
9743+
* @group post
9744+
*/
9745+
public function test_mapping_ep_asciifolding_filter() {
9746+
$indexable = ElasticPress\Indexables::factory()->get( 'post' );
9747+
$index_name = $indexable->get_index_name();
9748+
$settings = ElasticPress\Elasticsearch::factory()->get_index_settings( $index_name );
9749+
$index_settings = $settings[ $index_name ]['settings'];
9750+
9751+
$es_version = ElasticPress\Elasticsearch::factory()->get_elasticsearch_version();
9752+
if ( version_compare( $es_version, '7.0', '<' ) ) {
9753+
// ES5 format: flattened keys like 'index.analysis.analyzer.default.filter.0', '.1', etc.
9754+
$default_filter_keys = array_filter(
9755+
$index_settings,
9756+
fn( $value, $key ) => str_contains( $key, 'index.analysis.analyzer.default.filter.' ),
9757+
ARRAY_FILTER_USE_BOTH
9758+
);
9759+
$this->assertContains( 'ep_asciifolding', $default_filter_keys );
9760+
9761+
$default_search_filter_keys = array_filter(
9762+
$index_settings,
9763+
fn( $value, $key ) => str_contains( $key, 'index.analysis.analyzer.default_search.filter.' ),
9764+
ARRAY_FILTER_USE_BOTH
9765+
);
9766+
$this->assertContains( 'ep_asciifolding', $default_search_filter_keys );
9767+
9768+
} else {
9769+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default.filter'] );
9770+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default_search.filter'] );
9771+
}
9772+
}
9773+
9774+
/**
9775+
* Test the post returns correct with accented characters
9776+
*
9777+
* @since 5.3.3
9778+
* @group post
9779+
*/
9780+
public function test_search_handles_accented_characters_correctly() {
9781+
$post_id_1 = $this->ep_factory->post->create( [ 'post_title' => 'Coöperàtîôn' ] );
9782+
$post_id_2 = $this->ep_factory->post->create( [ 'post_title' => 'Fiancéé' ] );
9783+
9784+
// disable fuzziness
9785+
add_filter( 'ep_fuzziness', '__return_zero' );
9786+
9787+
ElasticPress\Elasticsearch::factory()->refresh_indices();
9788+
9789+
$query = new \WP_Query(
9790+
[
9791+
's' => 'coöperàtîôn',
9792+
]
9793+
);
9794+
$this->assertTrue( $query->elasticsearch_success );
9795+
$this->assertEquals( 1, $query->found_posts );
9796+
$this->assertEquals( $post_id_1, $query->posts[0]->ID );
9797+
9798+
$query = new \WP_Query(
9799+
[
9800+
's' => 'fiancee',
9801+
]
9802+
);
9803+
$this->assertTrue( $query->elasticsearch_success );
9804+
$this->assertEquals( 1, $query->found_posts );
9805+
$this->assertEquals( $post_id_2, $query->posts[0]->ID );
9806+
}
9807+
97399808
/**
97409809
* Test if aggregations are set
97419810
*

‎tests/php/indexables/TestTerm.php‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1899,4 +1899,20 @@ public function test_mapping_ep_stop_filter() {
18991899
$index_settings = $settings[ $index_name ]['settings'];
19001900
$this->assertSame( '_arabic_', $index_settings['index.analysis.filter.ep_stop.stopwords'] );
19011901
}
1902+
1903+
/**
1904+
* Test the `ep_asciifolding` filter
1905+
*
1906+
* @since 5.3.3
1907+
* @group term
1908+
*/
1909+
public function test_mapping_ep_asciifolding_filter() {
1910+
$indexable = ElasticPress\Indexables::factory()->get( 'term' );
1911+
$index_name = $indexable->get_index_name();
1912+
$settings = ElasticPress\Elasticsearch::factory()->get_index_settings( $index_name );
1913+
$index_settings = $settings[ $index_name ]['settings'];
1914+
1915+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default.filter'] );
1916+
$this->assertContains( 'ep_asciifolding', $index_settings['index.analysis.analyzer.default_search.filter'] );
1917+
}
19021918
}

0 commit comments

Comments
 (0)