Support possessive repeats for backreferences

Author: zherczeg

src/pcre2_compile.c

@@ -8066,6 +8066,13 @@ for (;; pptr++)
         tempcode += GET(tempcode, 1);
         break;
 #endif
+
+        case OP_REF:
+        case OP_REFI:
+        case OP_DNREF:
+        case OP_DNREFI:
+        tempcode += PRIV(OP_lengths)[*tempcode];
+        break;
         }
 
       /* If tempcode is equal to code (which points to the end of the repeated

src/pcre2_jit_compile.c

@@ -321,6 +321,7 @@ typedef struct ref_iterator_backtrack {
   backtrack_common common;
   /* Next iteration. */
   struct sljit_label *matchingpath;
+  BOOL possessive_or_exact;
 } ref_iterator_backtrack;
 
 typedef struct recurse_entry {
@@ -1142,7 +1143,11 @@ cc += PRIV(OP_lengths)[*cc];
 /* Although do_casefulcmp() uses only one local, the allocate_stack()
 calls during the repeat destroys LOCAL1 variables. */
 if (*cc >= OP_CRSTAR && *cc <= OP_CRPOSRANGE)
+  {
   locals_size += 2 * SSIZE_OF(sw);
+  if (*cc >= OP_CRPOSRANGE && GET2(cc, 1 + IMM2_SIZE + 1) != GET2(cc, 1 + IMM2_SIZE + 1 + IMM2_SIZE))
+    locals_size += SSIZE_OF(sw);
+  }
 
 return (current_locals_size >= locals_size) ? current_locals_size : locals_size;
 }
@@ -8390,9 +8395,10 @@ int offset = 0;
 struct sljit_label *label;
 struct sljit_jump *zerolength;
 struct sljit_jump *jump = NULL;
+jump_list *match_failed = NULL;
 PCRE2_SPTR ccbegin = cc;
 int min = 0, max = 0;
-BOOL minimize;
+BOOL minimize, exact;
 
 PUSH_BACKTRACK(sizeof(ref_iterator_backtrack), cc, NULL);
 
@@ -8415,38 +8421,137 @@ type = cc[1 + IMM2_SIZE];
 SLJIT_COMPILE_ASSERT((OP_CRSTAR & 0x1) == 0, crstar_opcode_must_be_even);
 /* Update ref_update_local_size() when this changes. */
 SLJIT_ASSERT(local_start + 2 * SSIZE_OF(sw) <= (int)LOCAL0 + common->locals_size);
-minimize = (type & 0x1) != 0;
+minimize = FALSE;
+exact = FALSE;
 switch(type)
   {
-  case OP_CRSTAR:
   case OP_CRMINSTAR:
+  minimize = TRUE;
+  PCRE2_FALLTHROUGH /* Fall through */
+  case OP_CRSTAR:
+  case OP_CRPOSSTAR:
   min = 0;
   max = 0;
   cc += 1 + IMM2_SIZE + 1;
   break;
-  case OP_CRPLUS:
+
   case OP_CRMINPLUS:
+  minimize = TRUE;
+  PCRE2_FALLTHROUGH /* Fall through */
+  case OP_CRPLUS:
+  case OP_CRPOSPLUS:
   min = 1;
   max = 0;
   cc += 1 + IMM2_SIZE + 1;
   break;
-  case OP_CRQUERY:
+
   case OP_CRMINQUERY:
+  minimize = TRUE;
+  PCRE2_FALLTHROUGH /* Fall through */
+  case OP_CRQUERY:
+  case OP_CRPOSQUERY:
   min = 0;
   max = 1;
   cc += 1 + IMM2_SIZE + 1;
   break;
-  case OP_CRRANGE:
+
   case OP_CRMINRANGE:
+  minimize = TRUE;
+  PCRE2_FALLTHROUGH /* Fall through */
+  case OP_CRRANGE:
+  case OP_CRPOSRANGE:
   min = GET2(cc, 1 + IMM2_SIZE + 1);
   max = GET2(cc, 1 + IMM2_SIZE + 1 + IMM2_SIZE);
+  SLJIT_ASSERT(min > 1 || max > 1);
+  if (min == max)
+    exact = TRUE;
   cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE;
   break;
   default:
   SLJIT_UNREACHABLE();
   break;
   }
 
+if (type >= OP_CRPOSSTAR || exact)
+  {
+  BACKTRACK_AS(ref_iterator_backtrack)->possessive_or_exact = TRUE;
+  if (ref)
+    {
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+    if (min > 0 && !common->unset_backref)
+      add_jump(compiler, &backtrack->own_backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)));
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1));
+    }
+  else
+    {
+    compile_dnref_search(common, ccbegin, min > 0 ? &backtrack->own_backtracks : NULL);
+    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start + SSIZE_OF(sw), TMP2, 0);
+    zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
+    }
+
+  if (exact)
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, SLJIT_IMM, min);
+  else if (type != OP_CRPOSSTAR)
+    {
+    /* STR_PTR is NULL before reaching min. */
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, min > 0 ? SLJIT_IMM : STR_PTR, 0);
+    if (type == OP_CRPOSRANGE)
+      {
+      SLJIT_ASSERT(local_start + 3 * SSIZE_OF(sw) <= (int)LOCAL0 + common->locals_size);
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start + 2 * SSIZE_OF(sw), SLJIT_IMM, 0);
+      }
+    }
+
+  label = LABEL();
+  if (!ref)
+    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), local_start + SSIZE_OF(sw));
+
+  if (type == OP_CRPOSSTAR)
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, STR_PTR, 0);
+
+  compile_ref_matchingpath(common, ccbegin, exact ? &backtrack->own_backtracks : &match_failed, FALSE, FALSE);
+
+  if (type == OP_CRPOSSTAR)
+    JUMPTO(SLJIT_JUMP, label);
+  else if (type == OP_CRPOSPLUS || type == OP_CRPOSQUERY)
+    {
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, STR_PTR, 0);
+    if (type == OP_CRPOSPLUS)
+      JUMPTO(SLJIT_JUMP, label);
+    }
+  else if (exact)
+    {
+    OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), local_start, SLJIT_MEM1(SLJIT_SP), local_start, SLJIT_IMM, 1);
+    JUMPTO(SLJIT_NOT_ZERO, label);
+    }
+  else
+    {
+    OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), local_start + 2 * SSIZE_OF(sw), SLJIT_IMM, 1);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start + 2 * SSIZE_OF(sw), TMP1, 0);
+    if (min > 0)
+      CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, label);
+    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, STR_PTR, 0);
+    if (max == 0)
+      JUMPTO(SLJIT_JUMP, label);
+    else
+      CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, max, label);
+    }
+
+  if (!exact)
+    {
+    set_jumps(match_failed, LABEL());
+    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), local_start);
+
+    if (type != OP_CRPOSSTAR)
+      add_jump(compiler, &backtrack->own_backtracks, CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0));
+    }
+
+  JUMPHERE(zerolength);
+  count_match(common);
+  return cc;
+  }
+
 if (!minimize)
   {
   if (min == 0)
@@ -8458,6 +8563,7 @@ if (!minimize)
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0);
     /* Temporary release of STR_PTR. */
     OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
+
     /* Handles both invalid and empty cases. Since the minimum repeat,
     is zero the invalid case is basically the same as an empty case. */
     if (ref)
@@ -8469,6 +8575,7 @@ if (!minimize)
       OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start + SSIZE_OF(sw), TMP2, 0);
       zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw));
       }
+
     /* Restore if not zero length. */
     OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw));
     }
@@ -8477,6 +8584,7 @@ if (!minimize)
     allocate_stack(common, 1);
     if (ref)
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset));
+
     OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0);
 
     if (ref)
@@ -8504,11 +8612,11 @@ if (!minimize)
 
   if (min > 1 || max > 1)
     {
-    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), local_start);
-    OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
+    OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), local_start, SLJIT_IMM, 1);
     OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), local_start, TMP1, 0);
     if (min > 1)
       CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, label);
+
     if (max > 1)
       {
       jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max);
@@ -12139,6 +12247,12 @@ PCRE2_UCHAR type;
 
 type = cc[PRIV(OP_lengths)[*cc]];
 
+if (CURRENT_AS(ref_iterator_backtrack)->possessive_or_exact)
+  {
+  set_jumps(current->own_backtracks, LABEL());
+  return;
+  }
+
 if ((type & 0x1) == 0)
   {
   /* Maximize case. */

src/pcre2_match.c

@@ -5284,6 +5284,9 @@ fprintf(stderr, "++ %2ld op=%3d %s\n", Fecode - mb->start_code, *Fecode,
       case OP_CRMINPLUS:
       case OP_CRQUERY:
       case OP_CRMINQUERY:
+      case OP_CRPOSSTAR:
+      case OP_CRPOSPLUS:
+      case OP_CRPOSQUERY:
       fc = *Fecode++ - OP_CRSTAR;
       Lmin = rep_min[fc];
       Lmax = rep_max[fc];
@@ -5292,6 +5295,7 @@ fprintf(stderr, "++ %2ld op=%3d %s\n", Fecode - mb->start_code, *Fecode,
 
       case OP_CRRANGE:
       case OP_CRMINRANGE:
+      case OP_CRPOSRANGE:
       Lmin = GET2(Fecode, 1);
       Lmax = GET2(Fecode, 1 + IMM2_SIZE);
       reptype = rep_typ[*Fecode - OP_CRSTAR];
@@ -5403,6 +5407,9 @@ fprintf(stderr, "++ %2ld op=%3d %s\n", Fecode - mb->start_code, *Fecode,
         Feptr += slength;
         }
 
+      /* No recursion if the repeat type is possessive. */
+      if (reptype == REPTYPE_POS) break;
+
       /* If the length matched for each repetition is the same as the length of
       the captured group, we can easily work backwards. This is the normal
       case. However, in caseless UTF-8 mode there are pairs of case-equivalent

testdata/testinput2

@@ -7640,6 +7640,47 @@ a)"xI
 
 # --------------
 
+# Test backreferences with possessive repeats
+
+/(..)\1*+/B
+  ab
+  abc
+  abababc
+
+/(..)\1++/B
+  abc
+  ababc
+  aababab
+  aabababc
+
+/(..)\1?+/B
+  aa
+  aac
+  ababab
+
+# The + is intentionally missing,
+# exact repeat is always possessive
+/(..)\1{3}/B
+  abababac
+  abababab
+  ababababc
+
+/(..)\1{2,}+/B
+  ababac
+  ababab
+  abababab
+  ababababab
+  abababababc
+
+/(..)\1{3,5}+/B
+  abababac
+  abababab
+  ababababab
+  abababababab
+  ababababababab
+
+# --------------
+
 # Attempt at full coverage of the substitution buffer-management code - not
 # just covering each line in each macro, but covering each instantiation of each
 # line in those macros.