Skip to content

Commit 148d0a6

Browse files
lumjjblumjjb
authored
add depsdev scanner implementation (#2385)
Signed-off-by: Brandon Lum <[email protected]>
1 parent 73fa293 commit 148d0a6

File tree

2 files changed

+376
-1
lines changed

2 files changed

+376
-1
lines changed

‎pkg/ingestor/parser/common/scanner/scanner.go‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@ import (
1919
"context"
2020
"fmt"
2121
"net/http"
22+
"time"
2223

24+
deps_dev_client "github.com/guacsec/guac/internal/client/depsdevclient"
2325
"github.com/guacsec/guac/pkg/assembler"
26+
"github.com/guacsec/guac/pkg/assembler/clients/generated"
2427
cd_certifier "github.com/guacsec/guac/pkg/certifier/clearlydefined"
2528
eol_certifier "github.com/guacsec/guac/pkg/certifier/eol"
2629
osv_certifier "github.com/guacsec/guac/pkg/certifier/osv"
@@ -114,8 +117,68 @@ func purlsLicenseScanWithClient(
114117
return certLegalIngest, hasSourceAtIngest, nil
115118
}
116119

120+
// PurlDepsDevScan scans the purls and returns for metadata linked to the repository
121+
// and returns the list of scorecards and sources associations it finds from Deps.dev
122+
// This generally takes about 300ms - 600ms. With tests including 1-30 PURLs.
117123
func PurlsDepsDevScan(ctx context.Context, purls []string) ([]assembler.CertifyScorecardIngest, []assembler.HasSourceAtIngest, error) {
118-
return nil, nil, fmt.Errorf("Unimplemented")
124+
certifyScorecards := []assembler.CertifyScorecardIngest{}
125+
hasSourceAts := []assembler.HasSourceAtIngest{}
126+
127+
ddc, err := deps_dev_client.NewDepsClient(ctx)
128+
if err != nil {
129+
return nil, nil, fmt.Errorf("unable to initialize deps.dev client: %w", err)
130+
131+
}
132+
133+
ddc.RetrieveVersionsAndProjects(ctx, purls)
134+
components, err := ddc.GetMetadata(ctx, purls)
135+
if err != nil {
136+
return nil, nil, fmt.Errorf("unable to get metadata for purls in deps.dev: %w", err)
137+
}
138+
139+
for _, c := range components {
140+
hasSourceAt := createDepsDevHasSourceAtIngest(c.CurrentPackage, c.Source, c.UpdateTime.UTC())
141+
scorecard := createDepsDevScorecardIngest(c.Source, c.Scorecard)
142+
if hasSourceAt != nil {
143+
hasSourceAts = append(hasSourceAts, *hasSourceAt)
144+
}
145+
if scorecard != nil {
146+
certifyScorecards = append(certifyScorecards, *scorecard)
147+
}
148+
}
149+
return certifyScorecards, hasSourceAts, nil
150+
}
151+
152+
func createDepsDevHasSourceAtIngest(pkg *generated.PkgInputSpec, src *generated.SourceInputSpec, knownSince time.Time) *assembler.HasSourceAtIngest {
153+
if pkg != nil && src != nil {
154+
return &assembler.HasSourceAtIngest{
155+
Pkg: pkg,
156+
PkgMatchFlag: generated.MatchFlags{
157+
Pkg: generated.PkgMatchTypeAllVersions,
158+
},
159+
Src: src,
160+
HasSourceAt: &generated.HasSourceAtInputSpec{
161+
KnownSince: knownSince,
162+
Justification: "collected via deps.dev",
163+
Collector: "ingest_depsdev_scanner",
164+
Origin: "deps.dev",
165+
},
166+
}
167+
}
168+
return nil
169+
}
170+
171+
func createDepsDevScorecardIngest(src *generated.SourceInputSpec, scorecard *generated.ScorecardInputSpec) *assembler.CertifyScorecardIngest {
172+
if src != nil && scorecard != nil {
173+
scorecard.Origin = "deps.dev"
174+
scorecard.Collector = "ingest_depsdev_scanner"
175+
176+
return &assembler.CertifyScorecardIngest{
177+
Source: src,
178+
Scorecard: scorecard,
179+
}
180+
}
181+
return nil
119182
}
120183

121184
// runQueryOnBatchedPurls runs EvaluateClearlyDefinedDefinition from the clearly defined

‎pkg/ingestor/parser/common/scanner/scanner_test.go‎

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -700,3 +700,315 @@ func parseEOLValue(value string) map[string]string {
700700
}
701701
return result
702702
}
703+
704+
func TestPurlsDepsDevScan(t *testing.T) {
705+
ctx := logging.WithLogger(context.Background())
706+
tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z")
707+
tests := []struct {
708+
name string
709+
purls []string
710+
wantCSs []assembler.CertifyScorecardIngest
711+
wantHSAs []assembler.HasSourceAtIngest
712+
wantErr bool
713+
}{
714+
{
715+
name: "no purl",
716+
purls: []string{""},
717+
wantCSs: []assembler.CertifyScorecardIngest{},
718+
wantHSAs: []assembler.HasSourceAtIngest{},
719+
wantErr: false,
720+
},
721+
{
722+
name: "valid maven/org.webjars.npm/a",
723+
purls: []string{"pkg:maven/org.webjars.npm/[email protected]"},
724+
wantCSs: []assembler.CertifyScorecardIngest{
725+
{
726+
Source: &generated.SourceInputSpec{
727+
Type: "git",
728+
Namespace: "github.com/alfateam",
729+
Name: "a",
730+
},
731+
Scorecard: &generated.ScorecardInputSpec{
732+
// Not including all checks since they are not stable for testing
733+
// and are not used anyway. As long as its a non-empty list.
734+
Checks: []generated.ScorecardCheckInputSpec{
735+
{
736+
Check: "Pinned-Dependencies",
737+
Score: -1,
738+
},
739+
{
740+
Check: "Token-Permissions",
741+
Score: -1,
742+
},
743+
{
744+
Check: "Maintained",
745+
Score: 0,
746+
},
747+
{
748+
Check: "Binary-Artifacts",
749+
Score: 10,
750+
},
751+
},
752+
AggregateScore: 3.0999999046325684,
753+
TimeScanned: tm,
754+
ScorecardVersion: "v5.0.0-94-g51f31c98",
755+
ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b",
756+
Origin: "deps.dev",
757+
Collector: "ingest_depsdev_scanner",
758+
},
759+
},
760+
},
761+
wantHSAs: []assembler.HasSourceAtIngest{
762+
{
763+
Pkg: &generated.PkgInputSpec{
764+
Type: "maven",
765+
Namespace: ptrfrom.String("org.webjars.npm"),
766+
Name: "a",
767+
Version: ptrfrom.String("2.1.2"),
768+
Subpath: ptrfrom.String(""),
769+
},
770+
PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
771+
Src: &generated.SourceInputSpec{
772+
Type: "git",
773+
Namespace: "github.com/alfateam",
774+
Name: "a",
775+
},
776+
HasSourceAt: &generated.HasSourceAtInputSpec{
777+
KnownSince: tm,
778+
Justification: "collected via deps.dev",
779+
Origin: "deps.dev",
780+
Collector: "ingest_depsdev_scanner",
781+
},
782+
},
783+
},
784+
wantErr: false,
785+
},
786+
{
787+
name: "non-existent purl",
788+
purls: []string{"pkg:notexist/ns/[email protected]"},
789+
wantCSs: []assembler.CertifyScorecardIngest{},
790+
wantHSAs: []assembler.HasSourceAtIngest{},
791+
wantErr: false,
792+
},
793+
{
794+
name: "malformed purl",
795+
purls: []string{"not-a-purl"},
796+
wantCSs: []assembler.CertifyScorecardIngest{},
797+
wantHSAs: []assembler.HasSourceAtIngest{},
798+
wantErr: false,
799+
},
800+
{
801+
name: "valid pypi/wheel-axle-runtime",
802+
purls: []string{"pkg:pypi/[email protected]"},
803+
wantCSs: []assembler.CertifyScorecardIngest{
804+
{
805+
Source: &generated.SourceInputSpec{
806+
Type: "git",
807+
Namespace: "github.com/karellen",
808+
Name: "wheel-axle-runtime",
809+
},
810+
Scorecard: &generated.ScorecardInputSpec{
811+
// Not including all checks since they are not stable for testing
812+
// and are not used anyway. As long as its a non-empty list.
813+
Checks: []generated.ScorecardCheckInputSpec{
814+
{
815+
Check: "Pinned-Dependencies",
816+
Score: 0,
817+
},
818+
{
819+
Check: "Token-Permissions",
820+
Score: 8,
821+
},
822+
{
823+
Check: "Maintained",
824+
Score: 1,
825+
},
826+
{
827+
Check: "Binary-Artifacts",
828+
Score: 8,
829+
},
830+
},
831+
AggregateScore: 3.700000047683716,
832+
TimeScanned: tm,
833+
ScorecardVersion: "v5.0.0-94-g51f31c98",
834+
ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b",
835+
Origin: "deps.dev",
836+
Collector: "ingest_depsdev_scanner",
837+
},
838+
},
839+
},
840+
wantHSAs: []assembler.HasSourceAtIngest{
841+
{
842+
Pkg: &generated.PkgInputSpec{
843+
Type: "pypi",
844+
Namespace: ptrfrom.String(""),
845+
Name: "wheel-axle-runtime",
846+
Version: ptrfrom.String("0.0.4"),
847+
Subpath: ptrfrom.String(""),
848+
},
849+
PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
850+
Src: &generated.SourceInputSpec{
851+
Type: "git",
852+
Namespace: "github.com/karellen",
853+
Name: "wheel-axle-runtime",
854+
},
855+
HasSourceAt: &generated.HasSourceAtInputSpec{
856+
KnownSince: tm,
857+
Justification: "collected via deps.dev",
858+
Origin: "deps.dev",
859+
Collector: "ingest_depsdev_scanner",
860+
},
861+
},
862+
},
863+
wantErr: false,
864+
},
865+
{
866+
name: "multiple valid PURLs",
867+
purls: []string{
868+
"pkg:maven/org.webjars.npm/[email protected]",
869+
"pkg:pypi/[email protected]",
870+
},
871+
wantCSs: []assembler.CertifyScorecardIngest{
872+
{
873+
Source: &generated.SourceInputSpec{
874+
Type: "git",
875+
Namespace: "github.com/alfateam",
876+
Name: "a",
877+
},
878+
Scorecard: &generated.ScorecardInputSpec{
879+
// Not including all checks since they are not stable for testing
880+
// and are not used anyway. As long as its a non-empty list.
881+
Checks: []generated.ScorecardCheckInputSpec{
882+
{
883+
Check: "Pinned-Dependencies",
884+
Score: -1,
885+
},
886+
{
887+
Check: "Token-Permissions",
888+
Score: -1,
889+
},
890+
{
891+
Check: "Maintained",
892+
Score: 0,
893+
},
894+
{
895+
Check: "Binary-Artifacts",
896+
Score: 10,
897+
},
898+
},
899+
AggregateScore: 3.0999999046325684,
900+
TimeScanned: tm,
901+
ScorecardVersion: "v5.0.0-94-g51f31c98",
902+
ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b",
903+
Origin: "deps.dev",
904+
Collector: "ingest_depsdev_scanner",
905+
},
906+
},
907+
{
908+
Source: &generated.SourceInputSpec{
909+
Type: "git",
910+
Namespace: "github.com/karellen",
911+
Name: "wheel-axle-runtime",
912+
},
913+
Scorecard: &generated.ScorecardInputSpec{
914+
// Not including all checks since they are not stable for testing
915+
// and are not used anyway. As long as its a non-empty list.
916+
Checks: []generated.ScorecardCheckInputSpec{
917+
{
918+
Check: "Pinned-Dependencies",
919+
Score: 0,
920+
},
921+
{
922+
Check: "Token-Permissions",
923+
Score: 8,
924+
},
925+
{
926+
Check: "Maintained",
927+
Score: 1,
928+
},
929+
{
930+
Check: "Binary-Artifacts",
931+
Score: 8,
932+
},
933+
},
934+
AggregateScore: 3.700000047683716,
935+
TimeScanned: tm,
936+
ScorecardVersion: "v5.0.0-94-g51f31c98",
937+
ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b",
938+
Origin: "deps.dev",
939+
Collector: "ingest_depsdev_scanner",
940+
},
941+
},
942+
},
943+
wantHSAs: []assembler.HasSourceAtIngest{
944+
{
945+
Pkg: &generated.PkgInputSpec{
946+
Type: "maven",
947+
Namespace: ptrfrom.String("org.webjars.npm"),
948+
Name: "a",
949+
Version: ptrfrom.String("2.1.2"),
950+
Subpath: ptrfrom.String(""),
951+
},
952+
PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
953+
Src: &generated.SourceInputSpec{
954+
Type: "git",
955+
Namespace: "github.com/alfateam",
956+
Name: "a",
957+
},
958+
HasSourceAt: &generated.HasSourceAtInputSpec{
959+
KnownSince: tm,
960+
Justification: "collected via deps.dev",
961+
Origin: "deps.dev",
962+
Collector: "ingest_depsdev_scanner",
963+
},
964+
},
965+
{
966+
Pkg: &generated.PkgInputSpec{
967+
Type: "pypi",
968+
Namespace: ptrfrom.String(""),
969+
Name: "wheel-axle-runtime",
970+
Version: ptrfrom.String("0.0.4"),
971+
Subpath: ptrfrom.String(""),
972+
},
973+
PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"},
974+
Src: &generated.SourceInputSpec{
975+
Type: "git",
976+
Namespace: "github.com/karellen",
977+
Name: "wheel-axle-runtime",
978+
},
979+
HasSourceAt: &generated.HasSourceAtInputSpec{
980+
KnownSince: tm,
981+
Justification: "collected via deps.dev",
982+
Origin: "deps.dev",
983+
Collector: "ingest_depsdev_scanner",
984+
},
985+
},
986+
},
987+
wantErr: false,
988+
},
989+
}
990+
for _, tt := range tests {
991+
t.Run(tt.name, func(t *testing.T) {
992+
gotCSs, gotHSAs, err := PurlsDepsDevScan(ctx, tt.purls)
993+
if (err != nil) != tt.wantErr {
994+
t.Errorf("PurlsToScan() error = %v, wantErr %v", err, tt.wantErr)
995+
return
996+
}
997+
if diff := cmp.Diff(
998+
tt.wantCSs,
999+
gotCSs,
1000+
cmpopts.IgnoreFields(generated.ScorecardInputSpec{},
1001+
"AggregateScore", "TimeScanned",
1002+
"ScorecardVersion", "ScorecardCommit", "DocumentRef"),
1003+
cmpopts.IgnoreTypes(generated.ScorecardCheckInputSpec{}),
1004+
); diff != "" {
1005+
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
1006+
}
1007+
if diff := cmp.Diff(tt.wantHSAs, gotHSAs,
1008+
cmpopts.IgnoreFields(generated.HasSourceAtInputSpec{}, "KnownSince", "DocumentRef"),
1009+
); diff != "" {
1010+
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
1011+
}
1012+
})
1013+
}
1014+
}

0 commit comments

Comments
 (0)