who's been spying route

package.json

@@ -14,6 +14,7 @@
     "leads": "npm run build:ensure && NODE_ENV=dev node compiled/scripts/leads.js"
   },
   "dependencies": {
+    "@koa/cors": "^3.1.0",
     "@mailchimp/mailchimp_marketing": "3.0.27",
     "@sentry/node": "5.27.3",
     "@sentry/tracing": "5.27.3",
@@ -42,7 +43,8 @@
     "pg": "8.2.1",
     "react": "17.0.1",
     "react-dom": "17.0.1",
-    "twit": "2.2.11"
+    "twit": "2.2.11",
+    "wappalyzer": "6.5.24"
   },
   "devDependencies": {
     "@types/axios": "0.14.0",

src/handlers/index.ts

@@ -5,13 +5,16 @@ import { parseUrl } from '../parse-url'
 import { extractFiles } from './extractFiles'
 import { getCurrentUser } from './getCurrentUser'
 import { getAllMentions } from './leads'
+import { getSpies } from './spy'
 import * as mailchimp from '../external-apis/mailchimp'
 import * as twitter from '../external-apis/twitter'
 import { saveFeedback, extractImageData } from '../models/feedback'
 
 module Handlers {
   export const leads = getAllMentions
 
+  export const spy = getSpies
+
   export const getWebsite = website.getWebsite
 
   export const getMe: Handler<{ photoUrl: null | string }> = async ctx => {

src/handlers/spy.ts

@@ -0,0 +1,70 @@
+// tslint:disable:no-expression-statement no-let
+const Wappalyzer = require('wappalyzer')
+import Router = require('koa-router')
+
+type Category = {
+  id: number
+  slug: string
+  name: string
+}
+
+type Technology = {
+  slug: string
+  name: string
+  confidence: number
+  version: string | null
+  icon: string
+  website: string
+  cpe: string | null
+  categories: ReadonlyArray<Category>
+}
+
+const options = {
+  debug: false,
+  delay: 0,
+  headers: {},
+  maxDepth: 2,
+  maxUrls: 10,
+  maxWait: 5000,
+  recursive: true,
+  probe: true,
+  userAgent: 'Wappalyzer',
+  htmlMaxCols: 2000,
+  htmlMaxRows: 2000,
+}
+
+const wappalyzer = new Wappalyzer(options)
+
+export async function getSpies(ctx: Router.IRouterContext): Promise<any> {
+  const params = ctx.query
+  const url = params && params.url
+
+  if (!url) {
+    return { statusCode: 422, body: JSON.stringify({ message: 'Missing attribute - url' }) }
+  }
+  let technology_names
+  try {
+    await wappalyzer.init()
+
+    // Optionally set additional request headers
+    const headers = {}
+    const site = await wappalyzer.open(url, headers)
+
+    // Optionally capture and output errors
+    site.on('error', console.error)
+    const results = await site.analyze()
+
+    const technologies: ReadonlyArray<Technology> = results.technologies
+
+    const analytics_technologies = technologies.filter((tech: Technology) => {
+      return tech.categories.some((category: Category) => category.name === 'Analytics')
+    })
+    technology_names = analytics_technologies.map((tech: Technology) => tech.name)
+    // console.log(JSON.stringify(technology_names, null, 2))
+  } catch (error) {
+    // console.error(error)
+    return { statusCode: 400, body: JSON.stringify({ message: 'something went wrong' }) }
+  }
+  await wappalyzer.destroy()
+  return { statusCode: 200, body: JSON.stringify({ spies: technology_names }) }
+}

src/router.ts

@@ -64,6 +64,7 @@ export function createRouter(withRouter: (router: Router) => Router = identity):
     .post('/logout', mappedHandlers.logout)
     .post('/subscribe', mappedHandlers.subscribe)
     .get('/leads', mappedHandlers.leads)
+    .get('/spy', mappedHandlers.spy)
     .get('/fail', () => {
       throw new Error('Failure!')
     })

src/server.ts

@@ -1,5 +1,6 @@
 // tslint:disable:no-expression-statement
 import Koa = require('koa')
+const cors = require('@koa/cors')
 import * as Router from 'koa-router'
 import bodyParser = require('koa-body')
 import session = require('koa-session')
@@ -24,7 +25,7 @@ export function createServer(withRouter?: (router: Router) => Router): Koa {
       return next()
     })
   }
-
+  server.use(cors())
   server.use(bodyParser({ multipart: true, jsonLimit: '50mb' }))
   server.use((cookieParser as any).default())