Skip to content

Commit 175847f

Browse files
adrianeboydadrianeboyd
authored
Support list values and INTERSECTS in Matcher (#8784)
* Support list values and IS_INTERSECT in Matcher * Support list values as token attributes for set operators, not just as pattern values. * Add `IS_INTERSECT` operator. * Fix incorrect `ISSUBSET` and `ISSUPERSET` in schema and docs. * Rename IS_INTERSECT to INTERSECTS
1 parent fbbbda1 commit 175847f

5 files changed

Lines changed: 106 additions & 21 deletions

File tree

‎spacy/matcher/matcher.pyx‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,7 @@ class _RegexPredicate:
845845

846846

847847
class _SetPredicate:
848-
operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET")
848+
operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS")
849849

850850
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None):
851851
self.i = i
@@ -868,14 +868,16 @@ class _SetPredicate:
868868
else:
869869
value = get_token_attr_for_matcher(token.c, self.attr)
870870

871-
if self.predicate in ("IS_SUBSET", "IS_SUPERSET"):
871+
if self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
872872
if self.attr == MORPH:
873873
# break up MORPH into individual Feat=Val values
874874
value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value))
875875
else:
876-
# IS_SUBSET for other attrs will be equivalent to "IN"
877-
# IS_SUPERSET will only match for other attrs with 0 or 1 values
878-
value = set([value])
876+
# treat a single value as a list
877+
if isinstance(value, (str, int)):
878+
value = set([get_string_id(value)])
879+
else:
880+
value = set(get_string_id(v) for v in value)
879881
if self.predicate == "IN":
880882
return value in self.value
881883
elif self.predicate == "NOT_IN":
@@ -884,6 +886,8 @@ class _SetPredicate:
884886
return value <= self.value
885887
elif self.predicate == "IS_SUPERSET":
886888
return value >= self.value
889+
elif self.predicate == "INTERSECTS":
890+
return bool(value & self.value)
887891

888892
def __repr__(self):
889893
return repr(("SetPredicate", self.i, self.attr, self.value, self.predicate))
@@ -928,6 +932,7 @@ def _get_extra_predicates(spec, extra_predicates, vocab):
928932
"NOT_IN": _SetPredicate,
929933
"IS_SUBSET": _SetPredicate,
930934
"IS_SUPERSET": _SetPredicate,
935+
"INTERSECTS": _SetPredicate,
931936
"==": _ComparisonPredicate,
932937
"!=": _ComparisonPredicate,
933938
">=": _ComparisonPredicate,

‎spacy/schemas.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ class TokenPatternString(BaseModel):
159159
NOT_IN: Optional[List[StrictStr]] = Field(None, alias="not_in")
160160
IS_SUBSET: Optional[List[StrictStr]] = Field(None, alias="is_subset")
161161
IS_SUPERSET: Optional[List[StrictStr]] = Field(None, alias="is_superset")
162+
INTERSECTS: Optional[List[StrictStr]] = Field(None, alias="intersects")
162163

163164
class Config:
164165
extra = "forbid"
@@ -175,8 +176,9 @@ class TokenPatternNumber(BaseModel):
175176
REGEX: Optional[StrictStr] = Field(None, alias="regex")
176177
IN: Optional[List[StrictInt]] = Field(None, alias="in")
177178
NOT_IN: Optional[List[StrictInt]] = Field(None, alias="not_in")
178-
ISSUBSET: Optional[List[StrictInt]] = Field(None, alias="issubset")
179-
ISSUPERSET: Optional[List[StrictInt]] = Field(None, alias="issuperset")
179+
IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset")
180+
IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset")
181+
INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects")
180182
EQ: Union[StrictInt, StrictFloat] = Field(None, alias="==")
181183
NEQ: Union[StrictInt, StrictFloat] = Field(None, alias="!=")
182184
GEQ: Union[StrictInt, StrictFloat] = Field(None, alias=">=")

‎spacy/tests/matcher/test_matcher_api.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,16 @@ def test_matcher_subset_value_operator(en_vocab):
270270
doc[0].tag_ = "A"
271271
assert len(matcher(doc)) == 0
272272

273+
# IS_SUBSET with a list value
274+
Token.set_extension("ext", default=[])
275+
matcher = Matcher(en_vocab)
276+
pattern = [{"_": {"ext": {"IS_SUBSET": ["A", "B"]}}}]
277+
matcher.add("M", [pattern])
278+
doc = Doc(en_vocab, words=["a", "b", "c"])
279+
doc[0]._.ext = ["A"]
280+
doc[1]._.ext = ["C", "D"]
281+
assert len(matcher(doc)) == 2
282+
273283

274284
def test_matcher_superset_value_operator(en_vocab):
275285
matcher = Matcher(en_vocab)
@@ -308,6 +318,72 @@ def test_matcher_superset_value_operator(en_vocab):
308318
doc[0].tag_ = "A"
309319
assert len(matcher(doc)) == 3
310320

321+
# IS_SUPERSET with a list value
322+
Token.set_extension("ext", default=[])
323+
matcher = Matcher(en_vocab)
324+
pattern = [{"_": {"ext": {"IS_SUPERSET": ["A"]}}}]
325+
matcher.add("M", [pattern])
326+
doc = Doc(en_vocab, words=["a", "b", "c"])
327+
doc[0]._.ext = ["A", "B"]
328+
assert len(matcher(doc)) == 1
329+
330+
331+
def test_matcher_intersect_value_operator(en_vocab):
332+
matcher = Matcher(en_vocab)
333+
pattern = [{"MORPH": {"INTERSECTS": ["Feat=Val", "Feat2=Val2", "Feat3=Val3"]}}]
334+
matcher.add("M", [pattern])
335+
doc = Doc(en_vocab, words=["a", "b", "c"])
336+
assert len(matcher(doc)) == 0
337+
doc[0].set_morph("Feat=Val")
338+
assert len(matcher(doc)) == 1
339+
doc[0].set_morph("Feat=Val|Feat2=Val2")
340+
assert len(matcher(doc)) == 1
341+
doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3")
342+
assert len(matcher(doc)) == 1
343+
doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3|Feat4=Val4")
344+
assert len(matcher(doc)) == 1
345+
346+
# INTERSECTS with a single value is the same as IN
347+
matcher = Matcher(en_vocab)
348+
pattern = [{"TAG": {"INTERSECTS": ["A", "B"]}}]
349+
matcher.add("M", [pattern])
350+
doc = Doc(en_vocab, words=["a", "b", "c"])
351+
doc[0].tag_ = "A"
352+
assert len(matcher(doc)) == 1
353+
354+
# INTERSECTS with an empty pattern list matches nothing
355+
matcher = Matcher(en_vocab)
356+
pattern = [{"TAG": {"INTERSECTS": []}}]
357+
matcher.add("M", [pattern])
358+
doc = Doc(en_vocab, words=["a", "b", "c"])
359+
doc[0].tag_ = "A"
360+
assert len(matcher(doc)) == 0
361+
362+
# INTERSECTS with a list value
363+
Token.set_extension("ext", default=[])
364+
matcher = Matcher(en_vocab)
365+
pattern = [{"_": {"ext": {"INTERSECTS": ["A", "C"]}}}]
366+
matcher.add("M", [pattern])
367+
doc = Doc(en_vocab, words=["a", "b", "c"])
368+
doc[0]._.ext = ["A", "B"]
369+
assert len(matcher(doc)) == 1
370+
371+
# INTERSECTS with an empty pattern list matches nothing
372+
matcher = Matcher(en_vocab)
373+
pattern = [{"_": {"ext": {"INTERSECTS": []}}}]
374+
matcher.add("M", [pattern])
375+
doc = Doc(en_vocab, words=["a", "b", "c"])
376+
doc[0]._.ext = ["A", "B"]
377+
assert len(matcher(doc)) == 0
378+
379+
# INTERSECTS with an empty value matches nothing
380+
matcher = Matcher(en_vocab)
381+
pattern = [{"_": {"ext": {"INTERSECTS": ["A", "B"]}}}]
382+
matcher.add("M", [pattern])
383+
doc = Doc(en_vocab, words=["a", "b", "c"])
384+
doc[0]._.ext = []
385+
assert len(matcher(doc)) == 0
386+
311387

312388
def test_matcher_morph_handling(en_vocab):
313389
# order of features in pattern doesn't matter

‎website/docs/api/matcher.md‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,14 @@ it compares to another value.
7777
> ]
7878
> ```
7979
80-
| Attribute | Description |
81-
| -------------------------- | ------------------------------------------------------------------------------------------------------- |
82-
| `IN` | Attribute value is member of a list. ~~Any~~ |
83-
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
84-
| `ISSUBSET` | Attribute values (for `MORPH`) are a subset of a list. ~~Any~~ |
85-
| `ISSUPERSET` | Attribute values (for `MORPH`) are a superset of a list. ~~Any~~ |
86-
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
80+
| Attribute | Description |
81+
| -------------------------- | -------------------------------------------------------------------------------------------------------- |
82+
| `IN` | Attribute value is member of a list. ~~Any~~ |
83+
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
84+
| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ |
85+
| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ |
86+
| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ |
87+
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
8788
8889
## Matcher.\_\_init\_\_ {#init tag="method"}
8990

‎website/docs/usage/rule-based-matching.md‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -240,13 +240,14 @@ following rich comparison attributes are available:
240240
> # "Number=Sing|Gender=Neut|Polite=Infm" will not match because it's a superset
241241
> ```
242242
243-
| Attribute | Description |
244-
| -------------------------- | ------------------------------------------------------------------------------------------------------- |
245-
| `IN` | Attribute value is member of a list. ~~Any~~ |
246-
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
247-
| `IS_SUBSET` | Attribute values (for `MORPH`) are a subset of a list. ~~Any~~ |
248-
| `IS_SUPERSET` | Attribute values (for `MORPH`) are a superset of a list. ~~Any~~ |
249-
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
243+
| Attribute | Description |
244+
| -------------------------- | --------------------------------------------------------------------------------------------------------- |
245+
| `IN` | Attribute value is member of a list. ~~Any~~ |
246+
| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ |
247+
| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ |
248+
| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ |
249+
| `INTERSECTS` | Attribute value (for `MORPH` or custom list attributes) has a non-empty intersection with a list. ~~Any~~ |
250+
| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ |
250251
251252
#### Regular expressions {#regex new="2.1"}
252253

0 commit comments

Comments
 (0)