Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@
"cssnano-preset-advanced": "latest",
"finepack": "latest",
"git-authors-cli": "latest",
"gulp": "4",
"gulp": "5",
"gulp-concat": "latest",
"gulp-postcss": "latest",
"gulp-uglify": "latest",
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-amazon/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@metascraper/helpers": "workspace:*"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-amazon/test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const { readFile } = require('fs/promises')
const { resolve } = require('path')
const test = require('ava')
const test = require('ava').default

const metascraper = require('metascraper')([
require('..')(),
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-amazon/test/test.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { test: validator } = require('..')

Expand Down
4 changes: 2 additions & 2 deletions packages/metascraper-audio/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
],
"dependencies": {
"@metascraper/helpers": "workspace:*",
"p-reflect": "~2.1.0"
"p-reflect": "~3.1.0"
},
"devDependencies": {
"async-listen": "latest",
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-audio/test/iframe.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

const cheerio = require('cheerio')
const test = require('ava')
const test = require('ava').default

const { runServer } = require('./helpers')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-audio/test/index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const createMetascraper = (...args) =>
require('metascraper')([require('../src')(...args)])
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-audio/test/providers.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const { readFile } = require('fs/promises')
const { resolve } = require('path')
const test = require('ava')
const test = require('ava').default

const createMetascraper = (...args) =>
require('metascraper')([require('..')(...args)])
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-bluesky/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@metascraper/helpers": "workspace:*"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-bluesky/test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const { readFile } = require('fs/promises')
const { resolve } = require('path')
const test = require('ava')
const test = require('ava').default

const metascraperBluesky = require('metascraper-bluesky')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-bluesky/test/test.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { test: validator } = require('..')

Expand Down
4 changes: 2 additions & 2 deletions packages/metascraper-defuddle/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@
"@metascraper/helpers": "workspace:*",
"async-memoize-one": "~1.2.1",
"debug-logfmt": "~1.4.12",
"defuddle": "~0.18.1",
"defuddle": "~0.19.0",
"linkedom": "~0.18.12"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-defuddle/test/cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// Isolated in its own file: the extractor memoizes a single result keyed by its
// arguments, so a colliding call from another concurrent test would mask the
// behaviour under test. A dedicated process keeps these two calls deterministic.
const test = require('ava')
const test = require('ava').default
const path = require('path')
const fs = require('fs')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-defuddle/test/index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default
const path = require('path')
const fs = require('fs')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-description/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"@metascraper/helpers": "workspace:*"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-description/test/index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const createMetascraperDescription = require('..')
const createMetascraper = require('metascraper')
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-dribbble/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"@metascraper/helpers": "workspace:*"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-dribbble/test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const { readFile } = require('fs/promises')
const { resolve } = require('path')
const test = require('ava')
const test = require('ava').default

const metascraperDribbble = require('metascraper-dribbble')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-dribbble/test/test.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { test: validator } = require('..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-feed/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"@metascraper/helpers": "workspace:*"
},
"devDependencies": {
"ava": "5"
"ava": "8"
},
"engines": {
"node": ">= 22"
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-feed/test/index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const createMetascraperFeed = require('..')
const createMetascraper = require('metascraper')
Expand Down
18 changes: 9 additions & 9 deletions packages/metascraper-helpers/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,32 +24,32 @@
"dependencies": {
"audio-extensions": "0.0.0",
"chrono-node": "2.9.1",
"condense-whitespace": "~2.0.0",
"condense-whitespace": "~3.0.0",
"data-uri-utils": "~1.0.13",
"debug-logfmt": "~1.4.7",
"entities": "~7.0.1",
"entities": "~8.0.0",
"file-extension": "~4.0.5",
"has-values": "~2.0.1",
"image-extensions": "~1.1.0",
"is-relative-url": "~3.0.0",
"is-relative-url": "~4.1.0",
"is-uri": "~1.2.14",
"iso-639-3": "~2.2.0",
"iso-639-3": "~3.0.1",
"isostring": "0.0.1",
"jsdom": "~29.1.1",
"jsonrepair": "~3.14.0",
"lodash": "~4.18.1",
"memoize-one": "~6.0.0",
"microsoft-capitalize": "~1.0.8",
"mime": "3",
"normalize-url": "6",
"re2": "~1.24.1",
"mime": "4",
"normalize-url": "9",
"re2": "~1.25.0",
"smartquotes": "~2.3.2",
"tldts": "~7.4.2",
"url-regex-safe": "~4.0.0",
"video-extensions": "1"
"video-extensions": "2"
},
"devDependencies": {
"ava": "5",
"ava": "8",
"cheerio": "latest"
},
"engines": {
Expand Down
18 changes: 11 additions & 7 deletions packages/metascraper-helpers/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@

const memoizeOne = require('memoize-one').default || require('memoize-one')
const debug = require('debug-logfmt')('metascraper:find-rule')
const condenseWhitespace = require('condense-whitespace')
const { getExtension: mimeExtension } = require('mime')
const condenseWhitespace = require('condense-whitespace').default
const mime = require('mime').default
const mimeExtension = mime.getExtension.bind(mime)
const capitalize = require('microsoft-capitalize')
const isRelativeUrl = require('is-relative-url')
const isRelativeUrl = require('is-relative-url').default
const fileExtension = require('file-extension')
const _normalizeUrl = require('normalize-url')
const _normalizeUrl = require('normalize-url').default
const { jsonrepair } = require('jsonrepair')
const smartquotes = require('smartquotes')
const { decodeHTML } = require('entities')
const iso6393 = require('iso-639-3/to-1')
const { iso6393To1: iso6393 } = require('iso-639-3/iso6393-to-1.js')
const dataUri = require('data-uri-utils')
const hasValues = require('has-values')
const chrono = require('chrono-node')
Expand Down Expand Up @@ -102,7 +103,7 @@ const audioExtensions = chain(require('audio-extensions'))
}, {})
.value()

const videoExtensions = chain(require('video-extensions'))
const videoExtensions = chain(require('video-extensions').default)
.reduce((acc, ext) => {
acc[ext] = VIDEO
return acc
Expand Down Expand Up @@ -153,7 +154,10 @@ const sanetizeUrl = (url, opts) =>

const normalizeUrl = (baseUrl, relativePath, opts) => {
try {
return sanetizeUrl(absoluteUrl(baseUrl, relativePath), opts)
const absolute = absoluteUrl(baseUrl, relativePath)
// normalize-url v9 no longer rejects `javascript:` URLs; keep them out
if (urlObject(absolute).protocol === 'javascript:') return undefined
return sanetizeUrl(absolute, opts)
} catch (_) {}
}

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/has.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { has } = require('..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const cheerio = require('cheerio')

const test = require('ava')
const test = require('ava').default

const {
$jsonld,
Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/is-url/absolute.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { isUrl } = require('../..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/is-url/relative.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { isUrl } = require('../..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/jsonld.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

const cheerio = require('cheerio')
const test = require('ava')
const test = require('ava').default

const { $jsonld, jsonld } = require('..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/load-iframe.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
const cheerio = require('cheerio')
const { spawn } = require('child_process')
const path = require('path')
const test = require('ava')
const test = require('ava').default

const { loadIframe } = require('..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/memoize-one.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { memoizeOne } = require('..')

Expand Down
2 changes: 1 addition & 1 deletion packages/metascraper-helpers/test/titleize.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'

const test = require('ava')
const test = require('ava').default

const { titleize } = require('..')

Expand Down
4 changes: 2 additions & 2 deletions packages/metascraper-iframe/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@
"got": "~11.8.6",
"lodash": "~4.18.1",
"oembed-spec": "~1.4.6",
"p-reflect": "~2.1.0"
"p-reflect": "~3.1.0"
},
"devDependencies": {
"ava": "5",
"ava": "8",
"cacheable-lookup": "6",
"cheerio": "latest"
},
Expand Down
16 changes: 9 additions & 7 deletions packages/metascraper-iframe/src/from-html.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const { normalizeUrl, memoizeOne } = require('@metascraper/helpers')
const { forEach, get } = require('lodash')
const pReflect = require('p-reflect')
const pReflect = require('p-reflect').default
const got = require('got')

const getOembedUrl = memoizeOne(
Expand All @@ -26,12 +26,14 @@ const getOembedUrl = memoizeOne(
memoizeOne.EqualityUrlAndHtmlDom(newArgs, oldArgs)
)

const fromHTML = ({ gotOpts }) => async ({ htmlDom, url, iframe }) => {
const oembedUrl = getOembedUrl(url, htmlDom, iframe)
if (!oembedUrl) return
const { value } = await pReflect(got(oembedUrl, gotOpts).json())
return get(value, 'html')
}
const fromHTML =
({ gotOpts }) =>
async ({ htmlDom, url, iframe }) => {
const oembedUrl = getOembedUrl(url, htmlDom, iframe)
if (!oembedUrl) return
const { value } = await pReflect(got(oembedUrl, gotOpts).json())
return get(value, 'html')
}

fromHTML.test = (url, $) => getOembedUrl(url, $) !== undefined

Expand Down
Loading