From 0aa2f1008b6d7e0c89874d52b7824c6cac200517 Mon Sep 17 00:00:00 2001 From: Yulun Wu Date: Fri, 28 Aug 2026 15:06:52 -0700 Subject: [PATCH] Add per-web-feature scoring Fills in scoreFeature, dividing by the tests at least one run has rather than the tests the manifest lists, and taking the interop minimum per test. --- lib/feature-level-interop.js | 45 ++++++++- test/feature-level-interop.js | 180 ++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 test/feature-level-interop.js diff --git a/lib/feature-level-interop.js b/lib/feature-level-interop.js index 85a77db4..c02e3877 100644 --- a/lib/feature-level-interop.js +++ b/lib/feature-level-interop.js @@ -5,10 +5,53 @@ * interop). */ +const resultTrees = require('./result-trees'); + // Scores one feature for each browser in |expectedBrowsers|, or undefined if no // run has any of its |tests|. function scoreFeature(runs, expectedBrowsers, tests) { - throw new Error('scoreFeature is not implemented'); + const browserTestFractionTotals = new Map( + expectedBrowsers.map(product => [product, 0])); + let interopTestFractionTotal = 0; + let testCount = 0; + + for (const test of tests) { + const scores = new Map(); + for (const run of runs) { + const results = resultTrees.findTestResults(run.tree, test); + if (results !== undefined) { + scores.set(run.browser_name, resultTrees.scoreTestResults(results)); + } + } + // A test no run has did not exist at this revision, so it counts against + // nobody. + if (scores.size === 0) { + continue; + } + testCount += 1; + + // A product whose run lacks the test scores 0 for it, and interop takes the + // lowest score per test rather than per feature. + let lowest = 1; + for (const product of expectedBrowsers) { + const score = scores.has(product) ? scores.get(product) : 0; + browserTestFractionTotals.set( + product, browserTestFractionTotals.get(product) + score); + lowest = Math.min(lowest, score); + } + interopTestFractionTotal += lowest; + } + + if (testCount === 0) { + return undefined; + } + + return { + scores: new Map(expectedBrowsers.map(product => + [product, browserTestFractionTotals.get(product) / testCount])), + interop: interopTestFractionTotal / testCount, + tests: testCount, + }; } // Scores every web feature in |featureTestMap| against |runs|. diff --git a/test/feature-level-interop.js b/test/feature-level-interop.js new file mode 100644 index 00000000..08c5adc3 --- /dev/null +++ b/test/feature-level-interop.js @@ -0,0 +1,180 @@ +'use strict'; + +const assert = require('chai').assert; + +const featureLevelInterop = require('../lib/feature-level-interop'); +const {TreeBuilder} = require('./lib/tree-builder'); + +// Builds the map that lib.runs.fetchWebFeaturesManifest returns. +function createFeatureTests(featureToTests) { + return new Map(Object.entries(featureToTests).map( + ([feature, tests]) => [feature, new Set(tests)])); +} + +// Builds a list of runs from a map of browser name to built tree. +function createRuns(browserToTree) { + return Object.entries(browserToTree).map( + ([browserName, tree]) => ({browser_name: browserName, tree})); +} + +describe('feature-level-interop.js', () => { + describe('scoreFeature', () => { + const expectedBrowsers = ['chrome', 'firefox']; + + it('scores each product as its fraction of the feature passed', () => { + const runs = createRuns({ + chrome: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('css/b.html', 'PASS') + .build(), + firefox: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('css/b.html', 'FAIL') + .build(), + }); + + const scored = featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/css/a.html', '/css/b.html'])); + + assert.deepEqual(scored.scores, + new Map([['chrome', 1], ['firefox', 0.5]])); + assert.equal(scored.interop, 0.5); + assert.equal(scored.tests, 2); + }); + + it('scores each product passing only what the other fails as zero interop', + () => { + const runs = createRuns({ + chrome: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('css/b.html', 'FAIL') + .build(), + firefox: new TreeBuilder() + .addTest('css/a.html', 'FAIL') + .addTest('css/b.html', 'PASS') + .build(), + }); + + const scored = featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/css/a.html', '/css/b.html'])); + + // The lowest per-browser mean would be 0.5; the mean of the per-test + // minima is 0, because neither test passes everywhere. + assert.equal(scored.interop, 0); + }); + + it('divides by the tests found rather than the tests listed', () => { + const runs = createRuns({ + chrome: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + firefox: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + }); + + const scored = featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/css/a.html', '/css/future.html'])); + + assert.equal(scored.tests, 1); + assert.deepEqual(scored.scores, + new Map([['chrome', 1], ['firefox', 1]])); + }); + + it('scores a product zero for a test only another product has', () => { + const runs = createRuns({ + chrome: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('css/b.html', 'PASS') + .build(), + firefox: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + }); + + const scored = featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/css/a.html', '/css/b.html'])); + + assert.equal(scored.tests, 2); + assert.deepEqual(scored.scores, + new Map([['chrome', 1], ['firefox', 0.5]])); + assert.equal(scored.interop, 0.5); + }); + + it('returns undefined when no run has any of the tests', () => { + const runs = createRuns({ + chrome: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + firefox: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + }); + + assert.isUndefined(featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/dom/gone.html']))); + }); + + it('takes the lowest subtest fraction for a test', () => { + const runs = createRuns({ + chrome: new TreeBuilder() + .addTest('css/a.html', 'OK') + .addSubtest('css/a.html', 'one', 'PASS') + .addSubtest('css/a.html', 'two', 'PASS') + .build(), + firefox: new TreeBuilder() + .addTest('css/a.html', 'OK') + .addSubtest('css/a.html', 'one', 'PASS') + .addSubtest('css/a.html', 'two', 'FAIL') + .build(), + }); + + const scored = featureLevelInterop.scoreFeature( + runs, expectedBrowsers, new Set(['/css/a.html'])); + + assert.equal(scored.interop, 0.5); + }); + }); + + describe('scoreRuns', () => { + const expectedBrowsers = ['chrome', 'firefox']; + + it('scores every feature the manifest lists', () => { + const runs = createRuns({ + chrome: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('x.html', 'FAIL') + .build(), + firefox: new TreeBuilder() + .addTest('css/a.html', 'PASS') + .addTest('x.html', 'FAIL') + .build(), + }); + + const scored = featureLevelInterop.scoreRuns(runs, expectedBrowsers, + createFeatureTests({grid: ['/css/a.html'], audio: ['/x.html']})); + + assert.deepEqual([...scored.keys()].sort(), ['audio', 'grid']); + assert.equal(scored.get('grid').interop, 1); + assert.equal(scored.get('audio').interop, 0); + }); + + it('leaves out a feature no run has any test for', () => { + const runs = createRuns({ + chrome: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + firefox: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + }); + + const scored = featureLevelInterop.scoreRuns(runs, expectedBrowsers, + createFeatureTests({grid: ['/css/a.html'], audio: ['/x.html']})); + + assert.deepEqual([...scored.keys()], ['grid']); + }); + + it('scores a test that two features list under both', () => { + const runs = createRuns({ + chrome: new TreeBuilder().addTest('css/a.html', 'PASS').build(), + firefox: new TreeBuilder().addTest('css/a.html', 'FAIL').build(), + }); + + const scored = featureLevelInterop.scoreRuns( + runs, expectedBrowsers, createFeatureTests({ + grid: ['/css/a.html'], + subgrid: ['/css/a.html'], + })); + + assert.deepEqual([...scored.keys()].sort(), ['grid', 'subgrid']); + assert.equal(scored.get('subgrid').interop, 0); + }); + }); +});