Added tests and implementation for getReferrersForPost method

Author: cmraible

ghost/core/core/server/services/stats/PostsStatsService.js

@@ -2,9 +2,9 @@ const logging = require('@tryghost/logging');
 const errors = require('@tryghost/errors');
 
 /**
- * @typedef {Object} TopPostsOptions
+ * @typedef {Object} StatsServiceOptions
  * @property {string} [order='free_members desc'] - field to order by (free_members, paid_members, or mrr) and direction
- * @property {number} [limit=20] - maximum number of results to return
+ * @property {number|string} [limit=20] - maximum number of results to return
  * @property {string} [date_from] - optional start date filter (YYYY-MM-DD)
  * @property {string} [date_to] - optional end date filter (YYYY-MM-DD)
  * @property {string} [timezone='UTC'] - optional timezone for date interpretation
@@ -19,6 +19,14 @@ const errors = require('@tryghost/errors');
  * @property {number} mrr - Total MRR from paid conversions attributed to this post
  */
 
+/**
+ * @typedef {Object} ReferrerStatsResult
+ * @property {string} source - The referrer source (e.g., domain)
+ * @property {number} free_members - Count of members who signed up via this post/referrer but did not have a paid conversion attributed to the same post/referrer
+ * @property {number} paid_members - Count of members whose paid conversion event was attributed to this post/referrer
+ * @property {number} mrr - Total MRR from paid conversions attributed to this post/referrer
+ */
+
 class PostsStatsService {
     /**
      * @param {object} deps
@@ -31,14 +39,13 @@ class PostsStatsService {
     /**
      * Get top posts by attribution metrics (free_members, paid_members, or mrr)
      *
-     * @param {TopPostsOptions} options
+     * @param {StatsServiceOptions} options
      * @returns {Promise<{data: TopPostResult[]}>} The top posts based on the requested attribution metric
      */
     async getTopPosts(options = {}) {
-        logging.info('TopPostsStatsService.getTopPosts called with options:', options);
         try {
             const order = options.order || 'free_members desc';
-            const limitRaw = Number.parseInt(options.limit, 10);
+            const limitRaw = Number.parseInt(String(options.limit ?? 20), 10); // Ensure options.limit is a string for parseInt
             const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? limitRaw : 20;
             const [orderField, orderDirection = 'desc'] = order.split(' ');
 
@@ -74,7 +81,6 @@ class PostsStatsService {
                 .leftJoin('paid', 'p.id', 'paid.post_id')
                 .leftJoin('mrr', 'p.id', 'mrr.post_id');
 
-            // Apply final ordering and limiting (Removed the WHERE clause filtering for activity)
             const results = await query
                 .orderBy(orderField, orderDirection)
                 .limit(limit);
@@ -87,142 +93,124 @@ class PostsStatsService {
     }
 
     /**
-     * Get referrers for a post
+     * Get referrers for a specific post by attribution metrics
      * @param {string} postId
-     * @param {TopPostsOptions} options
-     * @returns {Promise<{data: TopPostResult[]}>} The referrers for the post
+     * @param {StatsServiceOptions} options
+     * @returns {Promise<{data: ReferrerStatsResult[]}>} The referrers for the post, ranked by the specified metric
      */
-    async getReferrersForPost(postId, options) {
-        const knex = this.knex;
-        const freeMembers = await knex('members_created_events as mce')
-            .select('mce.referrer_source as source')
-            .countDistinct('mce.member_id as free_members')
-            .leftJoin('members_subscription_created_events as msce', function () {
-                this.on('mce.member_id', '=', 'msce.member_id')
-                    .andOn('mce.attribution_id', '=', 'msce.attribution_id')
-                    .andOnVal('msce.attribution_type', '=', 'post');
-            })
-            .where('mce.attribution_id', postId)
-            .where('mce.attribution_type', 'post')
-            .whereNull('msce.id')
-            .groupBy('mce.referrer_source');
-
-        const paidMembers = await knex('members_subscription_created_events as msce')
-            .select('msce.referrer_source as source')
-            .countDistinct('msce.member_id as paid_members')
-            .where('msce.attribution_id', postId)
-            .where('msce.attribution_type', 'post')
-            .groupBy('msce.referrer_source');
+    async getReferrersForPost(postId, options = {}) {
+        try {
+            const order = options.order || 'free_members desc';
+            const limitRaw = Number.parseInt(String(options.limit ?? 20), 10);
+            const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? limitRaw : 20;
+            const [orderField, orderDirection = 'desc'] = order.split(' ');
 
-        const mrr = await knex('members_subscription_created_events as msce')
-            .select('msce.referrer_source as source')
-            .sum('mpse.mrr_delta as mrr')
-            .join('members_paid_subscription_events as mpse', function () {
-                this.on('mpse.subscription_id', '=', 'msce.subscription_id')
-                    .andOn('mpse.member_id', '=', 'msce.member_id');
-            })
-            .where('msce.attribution_id', postId)
-            .where('msce.attribution_type', 'post')
-            .groupBy('msce.referrer_source');
+            if (!['free_members', 'paid_members', 'mrr'].includes(orderField)) {
+                throw new errors.BadRequestError({
+                    message: `Invalid order field: ${orderField}. Must be one of: free_members, paid_members, mrr`
+                });
+            }
+            if (!['asc', 'desc'].includes(orderDirection.toLowerCase())) {
+                throw new errors.BadRequestError({
+                    message: `Invalid order direction: ${orderDirection}`
+                });
+            }
 
-        const map = new Map();
-        for (const row of freeMembers) {
-            map.set(row.source, {
-                source: row.source,
-                free_members: row.free_members,
-                paid_members: 0,
-                mrr: 0
-            });
-        }
+            const freeReferrersCTE = this._buildFreeReferrersSubquery(postId, options);
+            const paidReferrersCTE = this._buildPaidReferrersSubquery(postId, options);
+            const mrrReferrersCTE = this._buildMrrReferrersSubquery(postId, options);
 
-        for (const row of paidMembers) {
-            const existing = map.get(row.source) ?? {
-                source: row.source,
-                free_members: 0,
-                paid_members: 0,
-                mrr: 0
-            };
-            existing.paid_members = row.paid_members;
-            map.set(row.source, existing);
-        }
+            const baseReferrersQuery = this.knex('members_created_events as mce')
+                .select('mce.referrer_source as source')
+                .where('mce.attribution_id', postId)
+                .where('mce.attribution_type', 'post')
+                .union((qb) => {
+                    qb.select('msce.referrer_source as source')
+                        .from('members_subscription_created_events as msce')
+                        .where('msce.attribution_id', postId)
+                        .where('msce.attribution_type', 'post');
+                });
 
-        for (const row of mrr) {
-            const existing = map.get(row.source) ?? {
-                source: row.source,
-                free_members: 0,
-                paid_members: 0,
-                mrr: 0
-            };
-            existing.mrr = row.mrr;
-            map.set(row.source, existing);
-        }
+            let query = this.knex
+                .with('free_referrers', freeReferrersCTE)
+                .with('paid_referrers', paidReferrersCTE)
+                .with('mrr_referrers', mrrReferrersCTE)
+                .with('all_referrers', baseReferrersQuery)
+                .select(
+                    'ar.source',
+                    this.knex.raw('COALESCE(fr.free_members, 0) as free_members'),
+                    this.knex.raw('COALESCE(pr.paid_members, 0) as paid_members'),
+                    this.knex.raw('COALESCE(mr.mrr, 0) as mrr')
+                )
+                .from('all_referrers as ar')
+                .leftJoin('free_referrers as fr', 'ar.source', 'fr.source')
+                .leftJoin('paid_referrers as pr', 'ar.source', 'pr.source')
+                .leftJoin('mrr_referrers as mr', 'ar.source', 'mr.source')
+                .whereNotNull('ar.source');
 
-        const results = [...map.values()].sort((a, b) => b.mrr - a.mrr);
+            const results = await query
+                .orderBy(orderField, orderDirection)
+                .limit(limit);
 
-        return {data: results};
+            return {data: results};
+        } catch (error) {
+            logging.error(`Error fetching referrers for post ${postId}:`, error);
+            return {data: []};
+        }
     }
 
     /**
-     * Build a subquery/CTE for free_members count
+     * Build a subquery/CTE for free_members count (Post-level)
      * (Signed up on Post, Paid Elsewhere/Never)
      * @private
-     * @param {TopPostsOptions} options
+     * @param {StatsServiceOptions} options
      * @returns {import('knex').Knex.QueryBuilder}
      */
     _buildFreeMembersSubquery(options) {
         const knex = this.knex;
-        // Find members who signed up via this post (mce.attribution_id)
-        // but did NOT have a paid conversion via the SAME post (msce.attribution_id)
         let subquery = knex('members_created_events as mce')
             .select('mce.attribution_id as post_id')
             .countDistinct('mce.member_id as free_members')
             .leftJoin('members_subscription_created_events as msce', function () {
                 this.on('mce.member_id', '=', 'msce.member_id')
-                    .andOn('mce.attribution_id', '=', 'msce.attribution_id') // Important: check paid conversion attributed to SAME post
+                    .andOn('mce.attribution_id', '=', 'msce.attribution_id')
                     .andOnVal('msce.attribution_type', '=', 'post');
             })
             .where('mce.attribution_type', 'post')
-            .whereNull('msce.id') // Keep only those where the left join found no matching paid conversion on the same post
+            .whereNull('msce.id')
             .groupBy('mce.attribution_id');
 
-        // Apply date filter to the signup event
         this._applyDateFilter(subquery, options, 'mce.created_at');
-
         return subquery;
     }
 
     /**
-     * Build a subquery/CTE for paid_members count
+     * Build a subquery/CTE for paid_members count (Post-level)
      * (Paid conversion attributed to this post)
      * @private
-     * @param {TopPostsOptions} options
+     * @param {StatsServiceOptions} options
      * @returns {import('knex').Knex.QueryBuilder}
      */
     _buildPaidMembersSubquery(options) {
         const knex = this.knex;
-        // Count distinct members for whom a paid conversion event (subscription creation)
-        // was attributed to this post_id.
         let subquery = knex('members_subscription_created_events as msce')
             .select('msce.attribution_id as post_id')
             .countDistinct('msce.member_id as paid_members')
             .where('msce.attribution_type', 'post')
             .groupBy('msce.attribution_id');
 
-        // Apply date filter to the paid conversion event timestamp
         this._applyDateFilter(subquery, options, 'msce.created_at');
-
         return subquery;
     }
 
     /**
-     * Build a subquery/CTE for mrr sum
+     * Build a subquery/CTE for mrr sum (Post-level)
      * (Paid Conversions Attributed to Post)
      * @private
-     * @param {TopPostsOptions} options
+     * @param {StatsServiceOptions} options
      * @returns {import('knex').Knex.QueryBuilder}
      */
     _buildMrrSubquery(options) {
-        // Logic remains the same: Sum MRR for all paid conversions attributed to the post
         let subquery = this.knex('members_subscription_created_events as msce')
             .select('msce.attribution_id as post_id')
             .sum('mpse.mrr_delta as mrr')
@@ -233,27 +221,126 @@ class PostsStatsService {
             .where('msce.attribution_type', 'post')
             .groupBy('msce.attribution_id');
 
+        this._applyDateFilter(subquery, options, 'msce.created_at');
+        return subquery;
+    }
+
+    // --- Subqueries for getReferrersForPost ---
+
+    /**
+     * Build subquery for free members count per referrer for a specific post.
+     * (Signed up via Post/Referrer, Did NOT convert via SAME Post/Referrer)
+     * @private
+     * @param {string} postId
+     * @param {StatsServiceOptions} options
+     * @returns {import('knex').Knex.QueryBuilder}
+     */
+    _buildFreeReferrersSubquery(postId, options) {
+        const knex = this.knex;
+
+        // Simpler approach mirroring _buildFreeMembersSubquery
+        let subquery = knex('members_created_events as mce')
+            .select('mce.referrer_source as source')
+            .countDistinct('mce.member_id as free_members')
+            .leftJoin('members_subscription_created_events as msce', function () {
+                this.on('mce.member_id', '=', 'msce.member_id')
+                    .andOn('mce.attribution_id', '=', 'msce.attribution_id') // Conversion must be for the SAME post
+                    .andOn('mce.referrer_source', '=', 'msce.referrer_source') // And the SAME referrer
+                    .andOnVal('msce.attribution_type', '=', 'post');
+            })
+            .where('mce.attribution_id', postId)
+            .where('mce.attribution_type', 'post')
+            .whereNull('msce.id') // Keep only signups where no matching paid conversion (same post/referrer) exists
+            .groupBy('mce.referrer_source');
+
+        this._applyDateFilter(subquery, options, 'mce.created_at'); // Filter based on signup time
+        return subquery;
+    }
+
+    /**
+    * Build subquery for paid members count per referrer for a specific post.
+    * (Paid conversion attributed to this Post/Referrer)
+    * @private
+    * @param {string} postId
+    * @param {StatsServiceOptions} options
+    * @returns {import('knex').Knex.QueryBuilder}
+    */
+    _buildPaidReferrersSubquery(postId, options) {
+        const knex = this.knex;
+        let subquery = knex('members_subscription_created_events as msce')
+            .select('msce.referrer_source as source')
+            .countDistinct('msce.member_id as paid_members')
+            .where('msce.attribution_id', postId)
+            .where('msce.attribution_type', 'post')
+            .groupBy('msce.referrer_source');
+
         // Apply date filter to the paid conversion event timestamp
         this._applyDateFilter(subquery, options, 'msce.created_at');
+        return subquery;
+    }
 
+    /**
+     * Build subquery for MRR sum per referrer for a specific post.
+     * (MRR from paid conversions attributed to this Post/Referrer)
+     * @private
+     * @param {string} postId
+     * @param {StatsServiceOptions} options
+     * @returns {import('knex').Knex.QueryBuilder}
+     */
+    _buildMrrReferrersSubquery(postId, options) {
+        const knex = this.knex;
+        let subquery = knex('members_subscription_created_events as msce')
+            .select('msce.referrer_source as source')
+            .sum('mpse.mrr_delta as mrr')
+            .join('members_paid_subscription_events as mpse', function () {
+                this.on('mpse.subscription_id', '=', 'msce.subscription_id');
+                // Ensure we join on member_id as well for accuracy if subscription_id isn't unique across members? (Safeguard)
+                this.andOn('mpse.member_id', '=', 'msce.member_id');
+            })
+            .where('msce.attribution_id', postId)
+            .where('msce.attribution_type', 'post')
+            .groupBy('msce.referrer_source');
+
+        // Apply date filter to the paid conversion event timestamp
+        this._applyDateFilter(subquery, options, 'msce.created_at');
         return subquery;
     }
 
     /**
      * Apply date filters to a query builder instance
      * @private
      * @param {import('knex').Knex.QueryBuilder} query
-     * @param {TopPostsOptions} options
-     * @param {string} [dateColumn='created_at'] - The date column to filter on
+     * @param {StatsServiceOptions} options
+     * @param {string} dateColumn - The date column to filter on
      */
-    _applyDateFilter(query, options, dateColumn = 'created_at') {
+    _applyDateFilter(query, options, dateColumn) {
         // Note: Timezone handling might require converting dates before querying,
         // depending on how created_at is stored (UTC assumed here).
         if (options.date_from) {
-            query.where(dateColumn, '>=', options.date_from);
+            try {
+                // Attempt to parse and validate the date
+                const fromDate = new Date(options.date_from);
+                if (!isNaN(fromDate.getTime())) {
+                    query.where(dateColumn, '>=', options.date_from);
+                } else {
+                    logging.warn(`Invalid date_from format: ${options.date_from}. Skipping filter.`);
+                }
+            } catch (e) {
+                logging.warn(`Error parsing date_from: ${options.date_from}. Skipping filter.`);
+            }
         }
         if (options.date_to) {
-            query.where(dateColumn, '<=', options.date_to + ' 23:59:59');
+            try {
+                const toDate = new Date(options.date_to);
+                if (!isNaN(toDate.getTime())) {
+                    // Include the whole day for the 'to' date
+                    query.where(dateColumn, '<=', options.date_to + ' 23:59:59');
+                } else {
+                    logging.warn(`Invalid date_to format: ${options.date_to}. Skipping filter.`);
+                }
+            } catch (e) {
+                logging.warn(`Error parsing date_to: ${options.date_to}. Skipping filter.`);
+            }
         }
     }
 }