Skip to content

Commit 2f1e236

Browse files
luotianqi777luotianqi777
authored
Merge pull request #21 from luotianqi777/dev-1.0.7
Dev 1.0.7
2 parents 4fd6f9f + 97b7a20 commit 2f1e236

30 files changed

+488
-207
lines changed

‎.github/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path}
8686
| `out` | `string` | Set the output file. The result defaults to json format. | `-out output.json` |
8787
| `db` | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json` |
8888
| `progress` | `bool` | Show the progress bar. | `-progress` |
89+
| `dedup` | `bool` | Same result deduplication | `-dedup` |
8990

9091
------
9192

@@ -142,7 +143,7 @@ opensca-cli -db db.json -path ${project_path}
142143

143144
OpenSCA is an open source project, we appreciate your help!
144145

145-
To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md).
146+
To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md).
146147

147148

148149

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path}
8484
| `out` | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式 | `-out output.json` |
8585
| `db` | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json` |
8686
| `progress` | `bool` | 显示进度条 | `-progress` |
87+
| `dedup` | `bool` | 相同组件去重 | `-dedup` |
8788

8889
---
8990

‎analyzer/engine/archive.go

+19-27
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"util/filter"
1818
"util/logs"
1919
"util/model"
20+
"util/temp"
2021

2122
"github.com/axgle/mahonia"
2223
"github.com/mholt/archiver"
@@ -39,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) {
3940
filepath = strings.ReplaceAll(filepath, `\`, `/`)
4041
// 目录树根
4142
root = model.NewDirTree()
42-
root.Path = path.Base(filepath)
4343
var walker archiver.Walker
4444
if filter.Tar(filepath) {
4545
walker = archiver.NewTar()
@@ -86,35 +86,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) {
8686
// 支持解析的文件
8787
root.AddFile(model.NewFileData(fileName, data))
8888
} else if filter.AllPkg(fileName) {
89-
// 支持检测的压缩包
90-
rootPath, _ := os.Executable()
91-
rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`))
92-
tempPath := path.Join(rootPath, ".temp_path")
93-
// 创建临时文件夹
94-
os.Mkdir(tempPath, os.ModeDir)
95-
targetPath := path.Join(tempPath, path.Base(fileName))
9689
// 将压缩包解压到本地
97-
if out, err := os.Create(targetPath); err == nil {
98-
_, err = out.Write(data)
99-
out.Close()
100-
if err != nil {
101-
return errors.WithStack(err)
102-
}
103-
// 获取当前目录树
104-
dir := root.GetDir(fileName)
105-
name := path.Base(fileName)
106-
if _, ok := dir.SubDir[name]; !ok {
107-
// 将压缩包的内容添加到当前目录树
108-
dir.DirList = append(dir.DirList, name)
109-
dir.SubDir[name] = e.unArchiveFile(targetPath)
110-
}
111-
// 删除压缩包
112-
if err = os.Remove(targetPath); err != nil {
90+
temp.DoInTempDir(func(tempdir string) {
91+
targetPath := path.Join(tempdir, path.Base(fileName))
92+
if out, err := os.Create(targetPath); err == nil {
93+
_, err = out.Write(data)
94+
out.Close()
95+
if err != nil {
96+
logs.Error(err)
97+
}
98+
// 获取当前目录树
99+
dir := root.GetDir(fileName)
100+
name := path.Base(fileName)
101+
if _, ok := dir.SubDir[name]; !ok {
102+
// 将压缩包的内容添加到当前目录树
103+
dir.DirList = append(dir.DirList, name)
104+
dir.SubDir[name] = e.unArchiveFile(targetPath)
105+
}
106+
} else {
113107
logs.Error(err)
114108
}
115-
} else {
116-
logs.Error(err)
117-
}
109+
})
118110
}
119111
}
120112
return nil

‎analyzer/engine/engine.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"analyzer/java"
2525
"analyzer/javascript"
2626
"analyzer/php"
27+
"analyzer/python"
2728
"analyzer/ruby"
2829
"analyzer/rust"
2930
)
@@ -43,6 +44,9 @@ func NewEngine() Engine {
4344
rust.New(),
4445
golang.New(),
4546
erlang.New(),
47+
// 暂不解析groovy文件
48+
// groovy.New(),
49+
python.New(),
4650
},
4751
}
4852
}
@@ -97,7 +101,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep
97101
// 获取漏洞
98102
taskInfo.Error = vuln.SearchVuln(depRoot)
99103
// 是否仅保留漏洞组件
100-
if args.OnlyVuln {
104+
if args.Config.OnlyVuln {
101105
root := model.NewDepTree(nil)
102106
q := model.NewQueue()
103107
q.Push(depRoot)

‎analyzer/engine/parse.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package engine
77

88
import (
99
"path"
10+
"strings"
1011
"util/filter"
1112
"util/model"
1213
)
@@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree)
3637
for _, d := range analyzer.ParseFiles(files) {
3738
depRoot.Children = append(depRoot.Children, d)
3839
d.Parent = depRoot
39-
if d.Name != "" && d.Version.Ok() {
40+
if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() {
4041
d.Path = path.Join(d.Path, d.Dependency.String())
4142
}
4243
// 标识为直接依赖
@@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree)
7374
for len(q) > 0 {
7475
n := q[0]
7576
q = append(q[1:], n.Children...)
76-
if n.Name == "" || !n.Version.Ok() {
77+
if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() {
7778
n.Move(n.Parent)
7879
}
7980
}

‎analyzer/java/analyzer.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type {
3535

3636
// CheckFile Check if it is a parsable file
3737
func (Analyzer) CheckFile(filename string) bool {
38-
return filter.JavaPom(filename)
38+
return filter.JavaPom(filename) || filter.GroovyGradle(filename)
3939
}
4040

4141
// pomTree pom文件树
@@ -182,9 +182,15 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) {
182182
p.Path = f.Name
183183
poms = append(poms, p)
184184
}
185+
if filter.GroovyGradle(f.Name) {
186+
dep := model.NewDepTree(nil)
187+
dep.Path = f.Name
188+
parseGradle(dep, f)
189+
deps = append(deps, dep)
190+
}
185191
}
186192
// 构建jar树
187-
deps = buildJarTree(jarMap)
193+
deps = append(deps, buildJarTree(jarMap)...)
188194
// 构建pom树
189195
deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...)
190196
return

‎analyzer/java/ext.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,14 @@ import (
1818
"util/enum/language"
1919
"util/logs"
2020
"util/model"
21+
"util/temp"
2122

2223
"github.com/pkg/errors"
2324
)
2425

2526
// MvnDepTree 调用mvn解析项目获取依赖树
2627
func MvnDepTree(path string, root *model.DepTree) {
27-
pwd, err := os.Getwd()
28-
if err != nil {
29-
logs.Error(err)
30-
return
31-
}
28+
pwd := temp.GetPwd()
3229
os.Chdir(path)
3330
cmd := exec.Command("mvn", "dependency:tree", "--fail-never")
3431
out, _ := cmd.CombinedOutput()

‎analyzer/java/gradle.go

+38-7
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import (
66
"encoding/json"
77
"os"
88
"os/exec"
9+
"regexp"
10+
"strings"
911
"util/enum/language"
1012
"util/logs"
1113
"util/model"
14+
"util/temp"
1215
)
1316

1417
//go:embed oss.gradle
@@ -26,14 +29,10 @@ type gradleDep struct {
2629

2730
// GradleDepTree 尝试获取 gradle 依赖树
2831
func GradleDepTree(dirpath string, root *model.DepTree) {
29-
pwd, err := os.Getwd()
30-
if err != nil {
31-
logs.Error(err)
32-
return
33-
}
32+
pwd := temp.GetPwd()
3433
os.Chdir(dirpath)
3534
// 复制 oss.gradle
36-
if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil {
35+
if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil {
3736
logs.Warn(err)
3837
return
3938
}
@@ -52,7 +51,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) {
5251
data := out[startIndex+len(startTag) : endIndex]
5352
out = out[endIndex+1:]
5453
gdep := &gradleDep{MapDep: model.NewDepTree(root)}
55-
err = json.Unmarshal(data, &gdep.Children)
54+
err := json.Unmarshal(data, &gdep.Children)
5655
if err != nil {
5756
logs.Warn(err)
5857
}
@@ -78,3 +77,35 @@ func GradleDepTree(dirpath string, root *model.DepTree) {
7877
}
7978
return
8079
}
80+
81+
// parseGradle parse *.gradle or *.gradle.kts
82+
func parseGradle(root *model.DepTree, file *model.FileInfo) {
83+
regexs := []*regexp.Regexp{
84+
regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`),
85+
regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`),
86+
regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`),
87+
}
88+
for _, line := range strings.Split(string(file.Data), "\n") {
89+
for _, re := range regexs {
90+
match := re.FindStringSubmatch(line)
91+
// 有捕获内容
92+
if len(match) == 4 &&
93+
// 不以注释开头
94+
!strings.HasPrefix(strings.TrimSpace(line), "/") &&
95+
// 不是测试组件
96+
!strings.Contains(strings.ToLower(line), "testimplementation") &&
97+
// 去掉非组件内容
98+
!strings.Contains(line, "//") {
99+
ver := model.NewVersion(match[3])
100+
// 版本号正常
101+
if ver.Ok() {
102+
dep := model.NewDepTree(root)
103+
dep.Vendor = match[1]
104+
dep.Name = match[2]
105+
dep.Version = ver
106+
break
107+
}
108+
}
109+
}
110+
}
111+
}

‎analyzer/java/pom.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string {
129129
return p.Version
130130
case "${project.groupId}", "${groupId}", "${pom.groupId}":
131131
return p.GroupId
132-
case "${project.artifactId}":
132+
case "${project.artifactId}", "${artifactId}", "${pom.artifactId}":
133133
return p.ArtifactId
134134
case "${project.parent.version}", "${parent.version}":
135135
return p.Parent.Version

‎analyzer/python/analyzer.go

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package python
2+
3+
import (
4+
"util/enum/language"
5+
"util/filter"
6+
"util/model"
7+
)
8+
9+
type Analyzer struct {
10+
}
11+
12+
func New() Analyzer {
13+
return Analyzer{}
14+
}
15+
16+
// GetLanguage get language of analyzer
17+
func (Analyzer) GetLanguage() language.Type {
18+
return language.Python
19+
}
20+
21+
// CheckFile check parsable file
22+
func (Analyzer) CheckFile(filename string) bool {
23+
return filter.PythonSetup(filename) ||
24+
filter.PythonPipfile(filename) ||
25+
filter.PythonPipfileLock(filename)
26+
}
27+
28+
// ParseFiles parse dependency from file
29+
func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree {
30+
deps := []*model.DepTree{}
31+
for _, f := range files {
32+
dep := model.NewDepTree(nil)
33+
dep.Path = f.Name
34+
if filter.PythonSetup(f.Name) {
35+
parseSetup(dep, f)
36+
} else if filter.PythonPipfile(f.Name) {
37+
parsePipfile(dep, f)
38+
} else if filter.PythonPipfileLock(f.Name) {
39+
parsePipfileLock(dep, f)
40+
}
41+
deps = append(deps, dep)
42+
}
43+
return deps
44+
}

‎analyzer/python/oss.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import re
2+
import sys
3+
import json
4+
5+
def parse_setup_py(setup_py_path):
6+
"""解析setup.py文件"""
7+
with open(setup_py_path, "r") as f:
8+
pass_func = lambda **x: x
9+
try:
10+
import distutils
11+
distutils.core.setup = pass_func
12+
except Exception:
13+
pass
14+
try:
15+
import setuptools
16+
setuptools.setup = pass_func
17+
except Exception:
18+
pass
19+
# 获取setup参数
20+
args = {}
21+
code = re.sub('(?<!\w)setup\(','args=setup(',f.read())
22+
code = code.replace('__file__','"{}"'.format(setup_py_path))
23+
exec(code, args)
24+
if 'args' in args:
25+
data = args['args']
26+
info = {}
27+
for k in ['name','version','license','packages','install_requires','requires']:
28+
if k in data:
29+
info[k] = data[k]
30+
print('oss_start<<{}>>oss_end'.format(json.dumps(info)))
31+
32+
if __name__ == "__main__":
33+
if len(sys.argv) > 1:
34+
parse_setup_py(sys.argv[1])

0 commit comments

Comments
 (0)