Skip to content

Commit 5d81320

Browse files
chemzqmchemzqm
committed
chore(extension): more tests
1 parent 91bd81d commit 5d81320

File tree

6 files changed

+111
-89
lines changed

6 files changed

+111
-89
lines changed

‎src/__tests__/modules/extensionDependency.test.ts

+47-11
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import fs from 'fs'
2-
import tar from 'tar'
32
import http, { Server } from 'http'
43
import os from 'os'
54
import path from 'path'
5+
import tar from 'tar'
66
import { URL } from 'url'
77
import { v4 as uuid } from 'uuid'
8-
import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, VersionInfo } from '../../extension/dependency'
8+
import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, validVersionInfo, VersionInfo } from '../../extension/dependency'
99
import { Dependencies } from '../../extension/installer'
10-
import { writeJson, remove, loadJson } from '../../util/fs'
10+
import { loadJson, remove, writeJson } from '../../util/fs'
1111
import helper, { getPort } from '../helper'
1212

1313
process.env.NO_PROXY = '*'
@@ -22,13 +22,34 @@ describe('utils', () => {
2222
expect(getRegistries(u).length).toBe(3)
2323
})
2424

25+
it('should check valid versionInfo', async () => {
26+
expect(validVersionInfo(null)).toBe(false)
27+
expect(validVersionInfo({ name: 3 })).toBe(false)
28+
expect(validVersionInfo({ name: 'name', version: '', dist: {} })).toBe(false)
29+
expect(validVersionInfo({
30+
name: 'name', version: '1.0.0', dist: {
31+
tarball: '',
32+
integrity: '',
33+
shasum: ''
34+
}
35+
})).toBe(true)
36+
})
37+
2538
it('should checkFileSha1', async () => {
2639
let not_exists = path.join(os.tmpdir(), 'not_exists')
2740
let checked = await checkFileSha1(not_exists, 'shasum')
2841
expect(checked).toBe(false)
2942
let tarfile = path.resolve(__dirname, '../test.tar.gz')
3043
checked = await checkFileSha1(tarfile, 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4')
3144
expect(checked).toBe(true)
45+
// throw on error
46+
let bigfile = path.join(os.tmpdir(), 'bigfile')
47+
let buf = Buffer.allocUnsafe(1024 * 1024)
48+
fs.writeFileSync(bigfile, buf)
49+
let p = checkFileSha1(bigfile, '')
50+
fs.unlinkSync(bigfile)
51+
let res = await p
52+
expect(res).toBe(false)
3253
})
3354

3455
it('should untar files', async () => {
@@ -259,21 +280,17 @@ describe('DependenciesInstaller', () => {
259280
await install.loadInfo(url, 'foo', 10)
260281
}
261282
await expect(fn()).rejects.toThrow(Error)
283+
fn = async () => {
284+
await install.loadInfo(url, 'bar')
285+
}
286+
await expect(fn()).rejects.toThrow(Error)
262287
})
263288

264289
it('should fetchInfos', async () => {
265290
addJsonData()
266291
let install = create()
267292
await install.fetchInfos({ a: '^0.0.1' })
268293
expect(install.resolvedInfos.size).toBe(4)
269-
expect(install.resolvedVersions).toEqual([
270-
{ name: 'a', requirement: '^0.0.1', version: '0.0.1' },
271-
{ name: 'b', requirement: '^1.0.0', version: '1.0.0' },
272-
{ name: 'c', requirement: '^2.0.0', version: '2.0.0' },
273-
{ name: 'b', requirement: '^2.0.0', version: '2.0.0' },
274-
{ name: 'd', requirement: '^1.0.0', version: '1.0.0' },
275-
{ name: 'd', requirement: '>= 0.0.1', version: '1.0.0' }
276-
])
277294
})
278295

279296
it('should linkDependencies', async () => {
@@ -305,6 +322,25 @@ describe('DependenciesInstaller', () => {
305322
fs.unlinkSync(res)
306323
})
307324

325+
it('should throw when unable to resolve version', async () => {
326+
let install = create()
327+
expect(() => {
328+
install.resolveVersion('foo', '^1.0.0')
329+
}).toThrow()
330+
install.resolvedInfos.set('foo', {
331+
name: 'foo',
332+
versions: {
333+
'2.0.0': {} as any
334+
}
335+
})
336+
expect(() => {
337+
install.resolveVersion('foo', '^1.0.0')
338+
}).toThrow()
339+
expect(() => {
340+
install.resolveVersion('foo', '^2.0.0')
341+
}).toThrow()
342+
})
343+
308344
it('should check exists and download items', async () => {
309345
let items: DependencyItem[] = []
310346
items.push({

‎src/__tests__/modules/extensions.test.ts

+3
Original file line numberDiff line numberDiff line change
@@ -207,9 +207,12 @@ describe('extensions', () => {
207207
let link = path.join(extensions.modulesFolder, 'test-link')
208208
fs.mkdirSync(folder, { recursive: true })
209209
fs.symlinkSync(folder, link)
210+
let cacheFolder = path.join(extensions.modulesFolder, '.cache')
211+
fs.mkdirSync(cacheFolder, { recursive: true })
210212
extensions.cleanModulesFolder()
211213
expect(fs.existsSync(folder)).toBe(false)
212214
expect(fs.existsSync(link)).toBe(false)
215+
expect(fs.existsSync(cacheFolder)).toBe(true)
213216
})
214217

215218
it('should install global extension', async () => {

‎src/extension/dependency.ts

+57-70
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import download from '../model/download'
99
import fetch, { FetchOptions } from '../model/fetch'
1010
import { concurrent } from '../util'
1111
import { loadJson, writeJson } from '../util/fs'
12+
import { objectLiteral } from '../util/is'
13+
import { Mutex } from '../util/mutex'
1214
const logger = require('../util/logger')('extension-dependency')
1315

1416
export interface Dependencies { [key: string]: string }
@@ -33,12 +35,6 @@ interface ModuleInfo {
3335
}
3436
}
3537

36-
interface ResolvedVersion {
37-
name: string
38-
requirement: string
39-
version: string
40-
}
41-
4238
export interface DependencyItem {
4339
name: string
4440
version: string
@@ -53,8 +49,10 @@ export interface DependencyItem {
5349

5450
const NPM_REGISTRY = new URL('https://registry.npmjs.org')
5551
const YARN_REGISTRY = new URL('https://registry.yarnpkg.com')
52+
const TAOBAO_REGISTRY = new URL('https://registry.npmmirror.com')
5653
const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild']
5754
const INFO_TIMEOUT = global.__TEST__ ? 100 : 10000
55+
const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000
5856

5957
function toFilename(item: DependencyItem): string {
6058
return `${item.name}.${item.version}.tgz`
@@ -73,14 +71,22 @@ export function getRegistries(registry: URL): URL[] {
7371
return urls
7472
}
7573

74+
export function validVersionInfo(info: any): info is VersionInfo {
75+
if (!info) return false
76+
if (typeof info.name !== 'string' || typeof info.version !== 'string' || !info.dist) return false
77+
let { tarball, integrity, shasum } = info.dist
78+
if (typeof tarball !== 'string' || typeof integrity !== 'string' || typeof shasum !== 'string') return false
79+
return true
80+
}
81+
7682
export function getModuleInfo(text: string): ModuleInfo {
7783
let obj
7884
try {
7985
obj = JSON.parse(text) as any
8086
} catch (e) {
8187
throw new Error(`Invalid JSON data, ${e}`)
8288
}
83-
if (typeof obj.name !== 'string' || obj.versions == null) throw new Error(`Invalid JSON data, name or versions not found`)
89+
if (typeof obj.name !== 'string' || !objectLiteral(obj.versions)) throw new Error(`Invalid JSON data, name or versions not found`)
8490
return {
8591
name: obj.name,
8692
latest: obj['dist-tags']?.latest,
@@ -109,7 +115,7 @@ export function readDependencies(directory: string): { [key: string]: string } {
109115
}
110116

111117
export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined {
112-
if (latest && semver.satisfies(latest, requirement)) return latest
118+
if (latest && validVersionInfo(versions[latest]) && semver.satisfies(latest, requirement)) return latest
113119
let sorted = semver.rsort(versions.filter(v => semver.valid(v, { includePrerelease: false })))
114120
for (let v of sorted) {
115121
if (semver.satisfies(v, requirement)) return v
@@ -137,7 +143,7 @@ export async function checkFileSha1(filepath: string, shasum: string): Promise<b
137143
if (!fs.existsSync(filepath)) return Promise.resolve(false)
138144
return new Promise(resolve => {
139145
const input = createReadStream(filepath)
140-
input.on('error', () => {
146+
input.on('error', e => {
141147
resolve(false)
142148
})
143149
input.on('readable', () => {
@@ -152,8 +158,9 @@ export async function checkFileSha1(filepath: string, shasum: string): Promise<b
152158
})
153159
}
154160

161+
const mutex = new Mutex()
162+
155163
export class DependenciesInstaller {
156-
public resolvedVersions: ResolvedVersion[] = []
157164
public resolvedInfos: Map<string, ModuleInfo> = new Map()
158165
private tokenSource: CancellationTokenSource = new CancellationTokenSource()
159166
constructor(
@@ -168,28 +175,29 @@ export class DependenciesInstaller {
168175
}
169176

170177
public async installDependencies(directory: string): Promise<void> {
171-
// TODO reuse resolved.json
172178
let dependencies = readDependencies(directory)
173179
// no need to install
174180
if (!dependencies || Object.keys(dependencies).length == 0) {
175181
this.onMessage(`No dependencies`)
176182
return
177183
}
178-
this.onMessage('Resolving dependencies.')
179-
await this.fetchInfos(dependencies)
180-
this.onMessage('Linking dependencies.')
181-
// create DependencyItems
182-
let items: DependencyItem[] = []
183-
this.linkDependencies(dependencies, items)
184-
if (items.length > 0) {
184+
this.onMessage('Waiting for install dependencies.')
185+
// TODO reuse resolved.json
186+
await mutex.use(async () => {
187+
this.onMessage('Resolving dependencies.')
188+
await this.fetchInfos(dependencies)
189+
this.onMessage('Linking dependencies.')
190+
// create DependencyItems
191+
let items: DependencyItem[] = []
192+
this.linkDependencies(dependencies, items)
185193
let filepath = path.join(directory, 'resolved.json')
186194
writeJson(filepath, items)
187-
}
188-
this.onMessage('Downloading dependencies.')
189-
await this.downloadItems(items)
190-
this.onMessage('Extract modules.')
191-
await this.extractDependencies(items, dependencies, directory)
192-
this.onMessage('Done')
195+
this.onMessage('Downloading dependencies.')
196+
await this.downloadItems(items)
197+
this.onMessage('Extract modules.')
198+
await this.extractDependencies(items, dependencies, directory)
199+
this.onMessage('Done')
200+
})
193201
}
194202

195203
public async extractDependencies(items: DependencyItem[], dependencies: Dependencies, directory: string): Promise<void> {
@@ -211,19 +219,12 @@ export class DependenciesInstaller {
211219
addToRoot(item)
212220
})
213221
rootPackages.clear()
214-
215-
let err: unknown
216-
await concurrent(rootItems, async item => {
217-
try {
218-
let filename = toFilename(item)
219-
let tarfile = path.join(this.dest, filename)
220-
let dest = path.join(directory, 'node_modules', item.name)
221-
await untar(dest, tarfile)
222-
} catch (e) {
223-
err = e
224-
}
225-
}, 5)
226-
if (err) throw err
222+
for (let item of rootItems) {
223+
let filename = toFilename(item)
224+
let tarfile = path.join(this.dest, filename)
225+
let dest = path.join(directory, 'node_modules', item.name)
226+
await untar(dest, tarfile)
227+
}
227228
for (let item of rootItems) {
228229
let folder = path.join(directory, 'node_modules', item.name)
229230
await this.extractFor(item, items, rootItems, folder)
@@ -250,25 +251,20 @@ export class DependenciesInstaller {
250251
}))
251252
newRoot.push(...rootItems)
252253
for (let item of deps.values()) {
253-
if (item.dependencies) {
254-
let dest = path.join(folder, 'node_modules', item.name)
255-
await this.extractFor(item, items, newRoot, dest)
256-
}
254+
let dest = path.join(folder, 'node_modules', item.name)
255+
await this.extractFor(item, items, newRoot, dest)
257256
}
258257
}
259258

260259
public linkDependencies(dependencies: Dependencies | undefined, items: DependencyItem[]): void {
261260
if (!dependencies) return
262261
for (let [name, requirement] of Object.entries(dependencies)) {
263-
let version = this.resolveVersion(name, requirement)
264-
let item = items.find(o => o.name === name && o.version === version)
262+
let versionInfo = this.resolveVersion(name, requirement)
263+
let item = items.find(o => o.name === name && o.version === versionInfo.version)
265264
if (item) {
266265
if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement)
267266
} else {
268-
let info = this.resolvedInfos.get(name)
269-
let versionInfo = info ? info.versions[version] : undefined
270-
if (!versionInfo) throw new Error(`Version data not found for "${name}@${version}"`)
271-
let { dist } = versionInfo
267+
let { dist, version } = versionInfo
272268
items.push({
273269
name,
274270
version,
@@ -283,15 +279,14 @@ export class DependenciesInstaller {
283279
}
284280
}
285281

286-
private resolveVersion(name: string, requirement: string): string {
287-
let item = this.resolvedVersions.find(o => o.name == name && o.requirement == requirement)
288-
if (item) return item.version
282+
public resolveVersion(name: string, requirement: string): VersionInfo {
289283
let info = this.resolvedInfos.get(name)
290284
if (info) {
291285
let version = getVersion(requirement, Object.keys(info.versions), info.latest)
292286
if (version) {
293-
this.resolvedVersions.push({ name, requirement, version })
294-
return version
287+
let versionInfo = info.versions[version]
288+
versionInfo.version = version
289+
if (validVersionInfo(versionInfo)) return versionInfo
295290
}
296291
}
297292
throw new Error(`No valid version found for "${name}" ${requirement}`)
@@ -300,25 +295,18 @@ export class DependenciesInstaller {
300295
/**
301296
* Recursive fetch
302297
*/
303-
public async fetchInfos(dependencies: Dependencies): Promise<void> {
304-
let keys = Object.keys(dependencies)
298+
public async fetchInfos(dependencies: Dependencies | undefined): Promise<void> {
299+
let keys = Object.keys(dependencies ?? {})
305300
if (keys.length === 0) return
306-
let fetched: Map<string, ModuleInfo> = new Map()
307301
await Promise.all(keys.map(key => {
308-
if (this.resolvedInfos.has(key)) {
309-
fetched.set(key, this.resolvedInfos.get(key))
310-
return Promise.resolve()
311-
}
302+
if (this.resolvedInfos.has(key)) return Promise.resolve()
312303
return this.loadInfo(this.registry, key, INFO_TIMEOUT).then(info => {
313-
fetched.set(key, info)
314304
this.resolvedInfos.set(key, info)
315305
})
316306
}))
317-
for (let [key, info] of fetched.entries()) {
318-
let requirement = dependencies[key]
319-
let version = this.resolveVersion(key, requirement)
320-
let deps = info.versions[version].dependencies
321-
if (deps) await this.fetchInfos(deps)
307+
for (let key of keys) {
308+
let versionInfo = this.resolveVersion(key, dependencies[key])
309+
await this.fetchInfos(versionInfo.dependencies)
322310
}
323311
}
324312

@@ -338,13 +326,13 @@ export class DependenciesInstaller {
338326
let onFinish = () => {
339327
res.set(filename, filepath)
340328
finished++
341-
this.onMessage(`Downloaded ${finished}/${total}`)
329+
this.onMessage(`Downloaded ${filename} ${finished}/${total}`)
342330
}
343331
if (checked) {
344332
onFinish()
345333
} else {
346334
// 5min timeout
347-
await this.download(new URL(item.resolved), filename, item.shasum, retry, global.__TEST__ ? 1000 : 5 * 60 * 1000)
335+
await this.download(new URL(item.resolved), filename, item.shasum, retry, DOWNLOAD_TIMEOUT)
348336
onFinish()
349337
}
350338
} catch (e) {
@@ -355,7 +343,7 @@ export class DependenciesInstaller {
355343
return res
356344
}
357345

358-
public async fetch(url: string | URL, options: FetchOptions = {}, retry = 1): Promise<any> {
346+
public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise<any> {
359347
for (let i = 0; i < retry; i++) {
360348
try {
361349
return await fetch(url, options, this.tokenSource.token)
@@ -381,8 +369,7 @@ export class DependenciesInstaller {
381369
this.onMessage(`Error on fetch ${url.hostname}/${name}: ${e}`)
382370
}
383371
}
384-
if (!info) throw new Error(`Unable to fetch info for "${name}"`)
385-
return info
372+
throw new Error(`Unable to fetch info for "${name}"`)
386373
}
387374

388375
public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise<string> {

0 commit comments

Comments
 (0)