fix: report empty previousUrl as undefined (#1526)

Author: metal-messiah

src/common/url/clean-url.js

@@ -8,10 +8,11 @@ var patternWithoutHash = /([^?#]*)().*/
 
 /**
  * Cleans a URL by removing the query string and fragment (hash portion).
- * @param {string} url - The original URL to be cleaned.
+ * @param {string} [url] - The original URL to be cleaned.
  * @param {boolean} [keepHash=false] - Whether to preserve the hash portion of the URL.
  * @returns {string} The cleaned URL.
  */
 export function cleanURL (url, keepHash) {
+  if (!url) return url
   return url.replace(keepHash ? patternWithHash : patternWithoutHash, '$1$2')
 }

src/features/soft_navigations/aggregate/initial-page-load-interaction.js

@@ -14,7 +14,8 @@ export class InitialPageLoadInteraction extends Interaction {
     super(agentRef, IPL_TRIGGER_NAME, 0, null)
     this.queueTime = agentRef.info.queueTime
     this.appTime = agentRef.info.applicationTime
-    this.oldURL = document.referrer
+    /** @type {string|undefined} we assign as undefined if no referrer value is available so that URL grouping is not applied to an empty string at ingest */
+    this.oldURL = document.referrer || undefined
   }
 
   get firstPaint () { return firstPaint.current.value }

src/features/spa/aggregate/interaction.js

@@ -33,7 +33,7 @@ export function Interaction (eventName, timestamp, url, routeName, onFinished, a
   attrs.initialPageURL = initialLocation
   attrs.oldRoute = routeName
   attrs.newURL = url
-  attrs.oldURL = eventName === 'initialPageLoad' ? document.referrer : url
+  attrs.oldURL = eventName === 'initialPageLoad' ? document.referrer || undefined : url // document referrer can return '' and flipper url grouping gets weird with empty strings. Pass undefined to skip flipper.
   attrs.custom = {}
   attrs.store = {}
 }

tests/specs/soft_navigations/payloads.e2e.js

@@ -27,12 +27,11 @@ describe('attribution tests', () => {
           await browser.testHandle.assetURL('soft_navigations/action-text.html', configWithDenyList)
         ).then(() => browser.waitForAgentLoad())
       ])
-      const documentReferrer = await browser.execute(() => document.referrer)
       const ipl = interactionHarvests[0].request.body[0]
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       ;[interactionHarvests] = await Promise.all([
         interactionsCapture.waitForResult({ totalCount: 2 }),
@@ -55,13 +54,12 @@ describe('attribution tests', () => {
           await browser.testHandle.assetURL('soft_navigations/action-text.html', configWithDenyList)
         ).then(() => browser.waitForAgentLoad())
       ])
-      const documentReferrer = await browser.execute(() => document.referrer)
       const ipl = interactionHarvests[0].request.body[0]
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.children.length).toEqual(0)
       expect(ipl.isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       ;[interactionHarvests] = await Promise.all([
         interactionsCapture.waitForResult({ totalCount: 2 }),
@@ -84,13 +82,12 @@ describe('attribution tests', () => {
           await browser.testHandle.assetURL('soft_navigations/action-text.html', configWithDenyList)
         ).then(() => browser.waitForAgentLoad())
       ])
-      const documentReferrer = await browser.execute(() => document.referrer)
       const ipl = interactionHarvests[0].request.body[0]
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.children.length).toEqual(0)
       expect(ipl.isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       ;[interactionHarvests] = await Promise.all([
         interactionsCapture.waitForResult({ totalCount: 2 }),
@@ -113,13 +110,12 @@ describe('attribution tests', () => {
           await browser.testHandle.assetURL('soft_navigations/action-text.html', configWithDenyList)
         ).then(() => browser.waitForAgentLoad())
       ])
-      const documentReferrer = await browser.execute(() => document.referrer)
       const ipl = interactionHarvests[0].request.body[0]
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.children.length).toEqual(0)
       expect(ipl.isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       ;[interactionHarvests] = await Promise.all([
         interactionsCapture.waitForResult({ totalCount: 2 }),
@@ -143,13 +139,11 @@ describe('attribution tests', () => {
           .then(() => $('body').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
-
       const [{ request: { body: [ipl] } }, { request: { body: [rc] } }] = interactionHarvests
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.navTiming).toEqual(expect.any(Object))
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
       if (browserMatch(supportsFirstPaint)) expect(ipl.firstPaint).toBeGreaterThan(0)
       else expect(ipl.firstPaint).toBeNull()
       expect(ipl.firstContentfulPaint).toBeGreaterThan(0)

tests/specs/spa/payloads.e2e.js

@@ -27,13 +27,12 @@ describe('attribution tests', () => {
           .then(() => $('#one').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].children.length).toEqual(1)
       expect(interactionEvents[1].children[0]).toMatchObject({
@@ -54,14 +53,13 @@ describe('attribution tests', () => {
           .then(() => $('#two').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].children.length).toEqual(0)
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].children.length).toEqual(1)
       expect(interactionEvents[1].children[0]).toMatchObject({
@@ -82,14 +80,13 @@ describe('attribution tests', () => {
           .then(() => $('#three').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].children.length).toEqual(0)
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].children.length).toEqual(1)
       expect(interactionEvents[1].children[0]).toMatchObject({
@@ -110,14 +107,13 @@ describe('attribution tests', () => {
           .then(() => $('body').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].children.length).toEqual(0)
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].children.length).toEqual(0)
     })
@@ -137,14 +133,13 @@ describe('attribution tests', () => {
       ])
       const receiptTime = Date.now()
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].children[0].path.startsWith('/1/')).toEqual(true)
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].id).toBeTruthy()
       expect(interactionEvents[1].id.match(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/)).toBeTruthy()
@@ -187,14 +182,13 @@ describe('attribution tests', () => {
       ])
       const receiptTime = Date.now()
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const interactionEvents = JSONPath({ path: '$.[*].request.body.[?(!!@ && @.type===\'interaction\')]', json: interactionHarvests })
       expect(interactionEvents.length).toEqual(2)
 
       expect(interactionEvents[0].trigger).toEqual('initialPageLoad')
       expect(interactionEvents[0].children[0].path.startsWith('/1/')).toEqual(true)
       expect(interactionEvents[0].isRouteChange).not.toBeTruthy()
-      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(interactionEvents[0].oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(interactionEvents[1].id).toBeTruthy()
       expect(interactionEvents[1].id.match(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/)).toBeTruthy()
@@ -235,15 +229,14 @@ describe('attribution tests', () => {
           .then(() => $('body').click())
       ])
 
-      const documentReferrer = await browser.execute(() => document.referrer)
       const [{ request: { body: [ipl] } }, { request: { body: [rc] } }] = interactionHarvests
 
       expect(ipl.trigger).toEqual('initialPageLoad')
       expect(ipl.navTiming).toEqual(expect.any(Object))
       if (browserMatch(supportsFirstPaint)) expect(ipl.firstPaint).toBeGreaterThan(0)
       else expect(ipl.firstPaint).toBeNull()
       expect(ipl.firstContentfulPaint).toBeGreaterThan(0)
-      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual(documentReferrer) // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      if (browserMatch(notIOS)) expect(ipl.oldURL).toEqual('') // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
 
       expect(rc.category).toEqual('Route change')
       expect(rc.navTiming).toBeNull()

tests/specs/spa/zonejs.e2e.js

@@ -50,8 +50,6 @@ describe('spa interactions with zonejs', () => {
       await browser.url(url).then(() => browser.waitForAgentLoad())
     ])
 
-    const referrer = await browser.execute(() => document.referrer)
-
     const event = interactionHarvests[0].request.body
       .find(ev => ev.type === 'interaction' && ev.trigger === 'initialPageLoad')
     expect(event).toEqual(expect.objectContaining({
@@ -60,7 +58,7 @@ describe('spa interactions with zonejs', () => {
       trigger: 'initialPageLoad',
       initialPageURL: url.slice(0, url.indexOf('?')),
       newURL: url.slice(0, url.indexOf('?')),
-      ...(browserMatch(notIOS) ? { oldURL: referrer } : {}), // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
+      ...(browserMatch(notIOS) ? { oldURL: '' } : {}), // ios on lambdatest appears to return the wrong value for referrer when using browser.execute, which breaks this test condition. Confirmed referrer behavior works in real env
       children: expect.arrayContaining([
         expect.objectContaining({
           domain: expect.stringContaining('bam-test-1.nr-local.net'),

tests/unit/features/soft_navigations/aggregate/initial-page-load-interaction.test.js

@@ -38,6 +38,6 @@ test('InitialPageLoad has correct oldURL', () => {
     runtime: { obfuscator: new Obfuscator({ init: { obfuscate: [] } }) }
   })
 
-  expect(ipl.oldURL).toBe(document.referrer || '')
+  expect(ipl.oldURL).toBe(document.referrer || undefined)
   expect(ipl.oldURL).not.toEqual(ipl.newURL)
 })

tools/wdio/args.mjs

@@ -68,7 +68,7 @@ const args = yargs(hideBin(process.argv))
   .describe('l', 'default loader to inject into test pages')
 
   .boolean('T')
-  .default('T', false)
+  .default('T', true)
   .alias('T', 'tunnel')
   .describe('T', 'Launch LambdaTest tunnel for this test run using process.env credentials')