Merge pull request #5 from autogram-is/ESM

Optional filter for UrlSet classes

Author: eaton

README.md

@@ -7,4 +7,51 @@ URL Tools is a helper library whose sole purpose is making that process just a l
 - `ParsedUrl`, a wrapper for the standard WHATWG `URL` class that mixes in the domain and subdomain parsing from [`tldts`](https://www.npmjs.com/package/tldts), and provides a convenience serializer that  preserves the object's individual properties, rather that flattening it to an href the way `URL.toJSON()` does.
 - A collection set of helper functions for URL filtering and normalizing operations, including sorting querystring parameters, stripping social sharing cruft, remapping 'ww1', 'ww2', etc. subdomains to a single canonical one, identifying web vs. non-web URLs, flagging urls on public hosting like S3, and more.
 - `NormalizedUrl`, a version of `ParsedUrl` that applies one of those normalizer functions automatically on instantiation. Functions that accept a `URL` or `ParsedURL` as a parameter can use `if (url instanceof NormalizedUrl)` to determine whether a given incoming URL has already been normalized. Yay.
-- `UrlSet`, `ParsedUrlSet`, and `NormalizedUrlSet`, a trio of Set classes that store, de-duplicate, and normalize piles of Urls in bulk. It's a bit fussy with bulk adding of relative URLs, but you can pass in a 'fallback base url' that helps in some circumstances.
+- `UrlSet`, `ParsedUrlSet`, and `NormalizedUrlSet`, a trio of Set classes that store, de-duplicate, filter, and normalize piles of Urls in bulk. It's a bit fussy with bulk adding of relative URLs, but you can pass in a 'fallback base url' that helps in some circumstances.
+
+## Installation
+
+`npm install @autogram/url-tools`
+
+## Usage
+
+UrlSet is the simplest example; toss URLs at it, and parsed URLs come out. Any that couldn't be parsed can be found the `urlSet.unparsable` property.
+
+```
+import { UrlSet } from '@autogram/url-tools';
+const rawUrls = [
+  'http://example.com',
+  'https://127.0.0.1',
+  'tel:1-800-555-1212',
+  'definitely-not-a-url'
+];
+
+const us = new UrlSet(rawUrls);
+for (url of us) {
+  console.log(url.href);
+}
+// 'http://example.com', 'https://127.0.0.1', 'tel:1-800-555-1212'
+
+console.log([...us.unparsable]); // ['definitely-not-a-url']
+```
+
+Both `ParsedUrlSet` and `NormalizedUrlSet` can accept a `UrlFilter` function in their constructor options; incoming URLs rejected by that function are shunted to the Set's `parsedUrlSet.rejected` property and not added to the Set proper. 
+
+`NormalizedUrlSet` can rely rely on NormalizedUrl's aggressive defaults, or pass in a UrlMutator function to use as an override.
+
+```
+import { NormalizedUrlSet, UrlFilters, UrlMutators } from '@autogram/url-tools';
+const options = {
+  filter: UrlFilters.isValidWebUrl,
+  normalizer: (u) => UrlMutators.forceProtocol(u, 'https')
+}
+
+const ns = new NormalizedUrlSet(rawUrls);
+for (n of ns) {
+  console.log(n.href);
+}
+// 'http://example.com', 'https://127.0.0.1'
+
+console.log([...ns.unparsable]); // ['definitely-not-a-url']
+console.log([...ns.rejected]);   // ['tel:1-800-555-1212']
+```

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autogram/url-tools",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Helper classes for validating, filtering, and normalizing URLs in bulk",
   "exports": "./dist/source/index.js",
   "types": "./dist/source/index.d.ts",

source/filters.ts

@@ -1,53 +1,44 @@
 import { ParsedUrl } from './parsed-url.js';
 import { regExpFromStringMatch, StringMatch } from './index.js';
 
-type UrlFilter = (url: ParsedUrl, options?: Record<string, unknown>) => boolean;
+export const isWebProtocol = function (url: ParsedUrl): boolean {
+  const webProtocols = ['http:', 'https:'];
+  return webProtocols.includes(url.protocol);
+};
 
-type MatchesPatternOptions = { pattern: StringMatch; property: string };
+export const isAuthenticated = function (url: ParsedUrl): boolean {
+  return (url.username + url.password).length > 0;
+};
 
-const UrlFilters = {
-  IsWebProtocol(url: ParsedUrl): boolean {
-    const webProtocols = ['http:', 'https:'];
-    return webProtocols.includes(url.protocol);
-  },
-  IsAuthenticated(url: ParsedUrl): boolean {
-    return (url.username + url.password).length > 0;
-  },
-  HasPublicSuffix(url: ParsedUrl): boolean {
-    return url.publicSuffix.length > 0;
-  },
-  MatchesPattern(
-    url: ParsedUrl,
-    options: Partial<MatchesPatternOptions> = {},
-  ): boolean {
-    options = {
-      pattern: [],
-      property: 'href',
-      ...options,
-    };
+export const hasPublicSuffix = function (url: ParsedUrl): boolean {
+  return url.publicSuffix.length > 0;
+};
 
-    if (options.property && options.property in url) {
-      const match = regExpFromStringMatch(options.pattern);
-      return match.test(url.properties[options.property].toString());
-    }
+export const matchesPattern = function (
+  url: ParsedUrl,
+  pattern: StringMatch = [],
+  property = 'href',
+): boolean {
+  if (pattern && property in url) {
+    const match = regExpFromStringMatch(pattern);
+    return match.test(url.properties[property].toString());
+  }
 
-    return false;
-  },
-  IsSocialShareLink(url: ParsedUrl): boolean {
-    return (
-      (url.domain === 'twitter.com' &&
-        url.pathname.startsWith('/intent/tweet')) || // Share links
-      (url.domain === 'pinterest.com' &&
-        url.pathname.startsWith('/pin/create/button')) ||
-      (url.domain === 'linkedin.com' &&
-        url.pathname.startsWith('/shareArticle')) ||
-      (url.domain === 'reddit.com' && url.pathname.startsWith('/submit')) ||
-      (url.domain === 'tumblr.com' &&
-        url.pathname.startsWith('/widgets/share')) ||
-      (url.domain === 'facebook.com' &&
-        url.pathname.startsWith('/sharer/sharer.php'))
-    );
-  },
+  return false;
 };
 
-export { UrlFilter, UrlFilters };
+export const isSocialShareLink = function (url: ParsedUrl): boolean {
+  return (
+    (url.domain === 'twitter.com' &&
+      url.pathname.startsWith('/intent/tweet')) || // Share links
+    (url.domain === 'pinterest.com' &&
+      url.pathname.startsWith('/pin/create/button')) ||
+    (url.domain === 'linkedin.com' &&
+      url.pathname.startsWith('/shareArticle')) ||
+    (url.domain === 'reddit.com' && url.pathname.startsWith('/submit')) ||
+    (url.domain === 'tumblr.com' &&
+      url.pathname.startsWith('/widgets/share')) ||
+    (url.domain === 'facebook.com' &&
+      url.pathname.startsWith('/sharer/sharer.php'))
+  );
+};

source/index.ts

@@ -1,13 +1,12 @@
-export { UrlMutator, UrlMutators } from './mutations.js';
-export { UrlFilter, UrlFilters } from './filters.js';
-export { ParsedUrl } from './parsed-url.js';
+export * as UrlMutators from './mutators.js';
+export * as UrlFilters from './filters.js';
+
+export { ParsedUrl, UrlFilter, UrlMutator } from './parsed-url.js';
 export { UrlSet, ParsedUrlSet, NormalizedUrlSet } from './url-set.js';
 export { NormalizedUrl } from './normalized-url.js';
 
 export type StringMatch = string | string[] | RegExp;
-export const regExpFromStringMatch = function (
-  pattern: StringMatch
-): RegExp {
+export const regExpFromStringMatch = function (pattern: StringMatch): RegExp {
   if (typeof pattern === 'string') {
     return new RegExp(pattern);
   }

source/mutations.ts

@@ -0,0 +1,95 @@
+import { ParsedUrl } from './parsed-url.js';
+import { StringMatch, regExpFromStringMatch } from './index.js';
+
+export const defaultNormalizer = function (url: ParsedUrl): ParsedUrl {
+  url = forceLowercaseHostname(url);
+  url = forceProtocol(url);
+  url = stripAuthentication(url);
+  url = stripAnchor(url);
+  url = stripIndexPages(url);
+  url = stripPort(url);
+  url = stripSubdomains(url);
+  url = stripTrailingSlash(url);
+  url = stripQueryParameters(url);
+  url = sortQueryParameters(url);
+  return url;
+};
+
+export const forceLowercaseHostname = function (url: ParsedUrl): ParsedUrl {
+  url.hostname = url.hostname.toLowerCase();
+  return url;
+};
+
+export const forceProtocol = function (
+  url: ParsedUrl,
+  options = 'https',
+): ParsedUrl {
+  url.protocol = options;
+  return url;
+};
+
+export const stripIndexPages = function (
+  url: ParsedUrl,
+  indexes?: string[],
+): ParsedUrl {
+  indexes = indexes ?? ['index.htm', 'index.html', 'default.aspx', 'index.php'];
+  for (const i of indexes) {
+    if (url.pathname.endsWith(i)) {
+      url.pathname = url.pathname.replace(i, '');
+    }
+  }
+
+  return url;
+};
+
+export const stripAnchor = function (url: ParsedUrl): ParsedUrl {
+  url.hash = '';
+  return url;
+};
+
+export const stripAuthentication = function (url: ParsedUrl): ParsedUrl {
+  url.username = '';
+  url.password = '';
+  return url;
+};
+
+export const stripPort = function (url: ParsedUrl): ParsedUrl {
+  url.port = '';
+  return url;
+};
+
+export const stripQueryParameters = function (
+  url: ParsedUrl,
+  options: StringMatch = /^utm_\s+|src|referrer|referer/,
+): ParsedUrl {
+  const stripList = regExpFromStringMatch(options);
+  for (const [name] of url.searchParams) {
+    if (stripList.test(name)) {
+      url.searchParams.delete(name);
+    }
+  }
+
+  return url;
+};
+
+export const stripSubdomains = function (
+  url: ParsedUrl,
+  options: StringMatch = /^ww[w\d]+/,
+): ParsedUrl {
+  const stripList = regExpFromStringMatch(options);
+  if (stripList.test(url.subdomain)) url.subdomain = '';
+  return url;
+};
+
+export const stripTrailingSlash = function (url: ParsedUrl): ParsedUrl {
+  if (url.pathname.endsWith('/')) {
+    url.pathname = url.pathname.slice(0, Math.max(0, url.pathname.length - 1));
+  }
+
+  return url;
+};
+
+export const sortQueryParameters = function (url: ParsedUrl): ParsedUrl {
+  url.searchParams.sort();
+  return url;
+};

source/mutators.ts

@@ -1,9 +1,10 @@
 import { URL } from 'node:url';
 import { ParsedUrl } from './parsed-url.js';
-import { UrlMutators, UrlMutator } from './mutations.js';
+import * as UrlMutators from './mutators.js';
+import { UrlMutator } from './index.js';
 
 export class NormalizedUrl extends ParsedUrl {
-  static normalizer: UrlMutator = UrlMutators.DefaultNormalizer;
+  static normalizer: UrlMutator = UrlMutators.defaultNormalizer;
   original: string;
 
   constructor(