Skip to content

Commit 875c3ed

Browse files
author
Matthew Robertson
authored
fix(cli): fix status check for includePassedAssertions (#595)
1 parent fb8acee commit 875c3ed

File tree

6 files changed

+308
-2
lines changed

6 files changed

+308
-2
lines changed

‎packages/cli/src/upload/upload.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,8 @@ async function runGithubStatusCheck(options, targetUrlMap) {
247247
for (const group of groupedResults) {
248248
const rawUrl = group[0].url;
249249
const urlLabel = getUrlLabelForGithub(rawUrl, options);
250-
const failedResults = group.filter(result => result.level === 'error');
251-
const warnResults = group.filter(result => result.level === 'warn');
250+
const failedResults = group.filter(result => result.level === 'error' && !result.passed);
251+
const warnResults = group.filter(result => result.level === 'warn' && !result.passed);
252252
const state = failedResults.length ? 'failure' : 'success';
253253
const context = getGitHubContext(urlLabel, options);
254254
const warningsLabel = warnResults.length ? ` with ${warnResults.length} warning(s)` : '';

‎packages/cli/test/autorun-github.test.js

+210
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
/**
2+
* @license Copyright 2021 Google Inc. All Rights Reserved.
3+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
4+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
5+
*/
6+
'use strict';
7+
8+
/* eslint-env jest */
9+
10+
const path = require('path');
11+
const rimraf = require('rimraf');
12+
const fs = require('fs');
13+
14+
const ApiClient = require('@lhci/utils/src/api-client.js');
15+
16+
const {
17+
createServer: createGithubServer,
18+
} = require('./fixtures/autorun-github/mock-github-server.js');
19+
const {runCLI, startServer, safeDeleteFile} = require('./test-utils.js');
20+
21+
describe('Lighthouse CI autorun CLI with GitHub status check', () => {
22+
const autorunDir = path.join(__dirname, 'fixtures/autorun-github');
23+
const lighthouseciDir = path.join(autorunDir, '.lighthouseci');
24+
25+
let server;
26+
let serverBaseUrl;
27+
let apiClient;
28+
let project;
29+
30+
let githubServer;
31+
32+
beforeAll(async () => {
33+
if (fs.existsSync(lighthouseciDir)) rimraf.sync(lighthouseciDir);
34+
githubServer = await createGithubServer();
35+
36+
server = await startServer();
37+
serverBaseUrl = `http://localhost:${server.port}/`;
38+
39+
apiClient = new ApiClient({rootURL: serverBaseUrl});
40+
project = await apiClient.createProject({name: 'Test'});
41+
});
42+
43+
afterAll(async () => {
44+
if (server) {
45+
server.process.kill();
46+
await safeDeleteFile(server.sqlFile);
47+
}
48+
if (githubServer) {
49+
await githubServer.close();
50+
}
51+
});
52+
53+
it('should submit status check to GitHub', async () => {
54+
const {stdout, stderr, status} = await runCLI(
55+
[
56+
'autorun',
57+
'--upload.githubToken=githubToken',
58+
`--upload.githubApiHost=http://localhost:${githubServer.port}`,
59+
`--upload.serverBaseUrl=${serverBaseUrl}`,
60+
`--upload.token=${project.token}`,
61+
],
62+
{
63+
cwd: autorunDir,
64+
env: {
65+
LHCI_BUILD_CONTEXT__CURRENT_HASH: 'hash',
66+
LHCI_BUILD_CONTEXT__GITHUB_REPO_SLUG: 'GoogleChrome/lighthouse-ci',
67+
LHCI_NO_LIGHTHOUSERC: undefined,
68+
},
69+
}
70+
);
71+
72+
expect(stdout).toMatchInlineSnapshot(`
73+
"✅ .lighthouseci/ directory writable
74+
✅ Configuration file found
75+
✅ Chrome installation found
76+
⚠️ GitHub token not set
77+
Healthcheck passed!
78+
79+
Automatically determined ./build as \`staticDistDir\`.
80+
Set it explicitly in lighthouserc.json if incorrect.
81+
82+
Started a web server on port XXXX...
83+
Running Lighthouse 1 time(s) on http://localhost:XXXX/bad.html
84+
Run #1...done.
85+
Running Lighthouse 1 time(s) on http://localhost:XXXX/good.html
86+
Run #1...done.
87+
Done running Lighthouse!
88+
89+
90+
Uploading median LHR of http://localhost:XXXX/bad.html...success!
91+
Open the report at https://storage.googleapis.com/lighthouse-infrastructure.appspot.com/reports/XXXX-XXXX.report.html
92+
Uploading median LHR of http://localhost:XXXX/good.html...success!
93+
Open the report at https://storage.googleapis.com/lighthouse-infrastructure.appspot.com/reports/XXXX-XXXX.report.html
94+
GitHub token found, attempting to set status...
95+
GitHub accepted \\"failure\\" status for \\"lhci/url/bad.html\\".
96+
GitHub accepted \\"success\\" status for \\"lhci/url/good.html\\".
97+
98+
"
99+
`);
100+
expect(stderr).toMatchInlineSnapshot(`
101+
"Checking assertions against 2 URL(s), 2 total run(s)
102+
103+
1 result(s) for http://localhost:XXXX/bad.html :
104+
105+
X viewport failure for minScore assertion
106+
Does not have a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
107+
https://web.dev/viewport/
108+
109+
expected: >=0.9
110+
found: 0
111+
all values: 0
112+
113+
1 result(s) for http://localhost:XXXX/good.html :
114+
115+
✅ viewport passing for minScore assertion
116+
Has a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
117+
https://web.dev/viewport/
118+
119+
expected: >=0.9
120+
found: 1
121+
all values: 1
122+
123+
Assertion failed. Exiting with status code 1.
124+
assert command failed. Exiting with status code 1.
125+
"
126+
`);
127+
expect(status).toEqual(1);
128+
}, 180000);
129+
130+
it('should submit status check to GitHub with invalid token', async () => {
131+
const {stdout, stderr, status} = await runCLI(
132+
[
133+
'autorun',
134+
'--upload.githubToken=invalidToken',
135+
`--upload.githubApiHost=http://localhost:${githubServer.port}`,
136+
`--upload.serverBaseUrl=${serverBaseUrl}`,
137+
`--upload.token=${project.token}`,
138+
],
139+
{
140+
cwd: autorunDir,
141+
env: {
142+
LHCI_BUILD_CONTEXT__CURRENT_HASH: 'hash',
143+
LHCI_BUILD_CONTEXT__GITHUB_REPO_SLUG: 'GoogleChrome/lighthouse-ci',
144+
LHCI_NO_LIGHTHOUSERC: undefined,
145+
},
146+
}
147+
);
148+
149+
expect(stdout).toMatchInlineSnapshot(`
150+
"✅ .lighthouseci/ directory writable
151+
✅ Configuration file found
152+
✅ Chrome installation found
153+
⚠️ GitHub token not set
154+
Healthcheck passed!
155+
156+
Automatically determined ./build as \`staticDistDir\`.
157+
Set it explicitly in lighthouserc.json if incorrect.
158+
159+
Started a web server on port XXXX...
160+
Running Lighthouse 1 time(s) on http://localhost:XXXX/bad.html
161+
Run #1...done.
162+
Running Lighthouse 1 time(s) on http://localhost:XXXX/good.html
163+
Run #1...done.
164+
Done running Lighthouse!
165+
166+
167+
Uploading median LHR of http://localhost:XXXX/bad.html...success!
168+
Open the report at https://storage.googleapis.com/lighthouse-infrastructure.appspot.com/reports/XXXX-XXXX.report.html
169+
Uploading median LHR of http://localhost:XXXX/good.html...success!
170+
Open the report at https://storage.googleapis.com/lighthouse-infrastructure.appspot.com/reports/XXXX-XXXX.report.html
171+
GitHub token found, attempting to set status...
172+
GitHub responded with 401
173+
174+
175+
GitHub responded with 401
176+
177+
178+
179+
"
180+
`);
181+
expect(stderr).toMatchInlineSnapshot(`
182+
"Checking assertions against 2 URL(s), 2 total run(s)
183+
184+
1 result(s) for http://localhost:XXXX/bad.html :
185+
186+
X viewport failure for minScore assertion
187+
Does not have a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
188+
https://web.dev/viewport/
189+
190+
expected: >=0.9
191+
found: 0
192+
all values: 0
193+
194+
1 result(s) for http://localhost:XXXX/good.html :
195+
196+
✅ viewport passing for minScore assertion
197+
Has a \`<meta name=\\"viewport\\">\` tag with \`width\` or \`initial-scale\`
198+
https://web.dev/viewport/
199+
200+
expected: >=0.9
201+
found: 1
202+
all values: 1
203+
204+
Assertion failed. Exiting with status code 1.
205+
assert command failed. Exiting with status code 1.
206+
"
207+
`);
208+
expect(status).toEqual(1);
209+
}, 180000);
210+
});

‎packages/cli/test/fixtures/autorun-github/build/bad.html

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8" />
5+
<title>bad test page for autorun usage</title>
6+
</head>
7+
<body>
8+
test
9+
</body>
10+
</html>

‎packages/cli/test/fixtures/autorun-github/build/good.html

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8" />
5+
<title>good test page for autorun usage</title>
6+
<meta name="viewport" content="width=device-width, initial-scale=1">
7+
</head>
8+
<body>
9+
test
10+
</body>
11+
</html>

‎packages/cli/test/fixtures/autorun-github/lighthouserc.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"ci": {
3+
"collect": {
4+
"numberOfRuns": 1
5+
},
6+
"assert": {
7+
"assertMatrix": [
8+
{
9+
"assertions": {
10+
"viewport": "error"
11+
}
12+
}
13+
],
14+
"includePassedAssertions": true
15+
},
16+
"upload": {
17+
"target": "temporary-public-storage"
18+
}
19+
}
20+
}

‎packages/cli/test/fixtures/autorun-github/mock-github-server.js

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/**
2+
* @license Copyright 2021 Google Inc. All Rights Reserved.
3+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
4+
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
5+
*/
6+
'use strict';
7+
const createHttpServer = require('http').createServer;
8+
const express = require('express');
9+
10+
/**
11+
* @return {Promise<{app: Parameters<typeof createHttpServer>[1]}>}
12+
*/
13+
async function createApp() {
14+
const app = express();
15+
16+
app.post('/repos/GoogleChrome/lighthouse-ci/statuses/hash', (req, res) => {
17+
if (!req.headers.authorization || req.headers.authorization !== 'token githubToken') {
18+
res.status(401);
19+
res.json();
20+
return;
21+
}
22+
23+
setTimeout(() => {
24+
res.status(201);
25+
res.json({});
26+
}, 100);
27+
});
28+
29+
return {app};
30+
}
31+
32+
/**
33+
* @return {Promise<{port: number, close: () => Promise<void>}>}
34+
*/
35+
async function createServer() {
36+
const {app} = await createApp();
37+
38+
return new Promise(resolve => {
39+
const server = createHttpServer(app);
40+
server.listen(0, () => {
41+
const serverAddress = server.address();
42+
const listenPort =
43+
typeof serverAddress === 'string' || !serverAddress ? 0 : serverAddress.port;
44+
45+
resolve({
46+
port: listenPort,
47+
close: async () => {
48+
await new Promise(r => server.close(r));
49+
},
50+
});
51+
});
52+
});
53+
}
54+
55+
module.exports = {createApp, createServer};

0 commit comments

Comments
 (0)