Adds HPSM and parsers for dependencies

Adds HPSM
Adds parser for go.sum, yarn.lock and package-lock.json
Adds tests for winnowing and new dependency parsers

Author: isasmendiagus

Diff for: package.json

@@ -1,6 +1,6 @@
 {
   "name": "scanoss",
-  "version": "0.2.28",
+  "version": "0.3.0",
   "description": "The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.",
   "main": "build/main/index.js",
   "typings": "build/main/index.d.ts",
@@ -18,16 +18,14 @@
     "fix": "run-s fix:*",
     "fix:prettier": "prettier \"src/**/*.ts\" --write",
     "fix:lint": "eslint src --ext .ts --fix",
-    "test": "run-s build test:*",
+    "test": "mocha -r ts-node/register 'tests/**/*.ts'",
     "test:lint": "eslint src --ext .ts",
     "test:prettier": "prettier \"src/**/*.ts\" --list-different",
     "test:spelling": "cspell \"{README.md,.github/*.md,src/**/*.ts}\"",
-    "test:unit": "nyc --silent ava",
     "check-cli": "run-s test diff-integration-tests check-integration-tests",
     "check-integration-tests": "run-s check-integration-test:*",
     "diff-integration-tests": "mkdir -p diff && rm -rf diff/test && cp -r test diff/test && rm -rf diff/test/test-*/.git && cd diff && git init --quiet && git add -A && git commit --quiet --no-verify --allow-empty -m 'WIP' && echo '\\n\\nCommitted most recent integration test output in the \"diff\" directory. Review the changes with \"cd diff && git diff HEAD\" or your preferred git diff viewer.'",
     "watch:build": "tsc -p tsconfig.json -w",
-    "watch:test": "nyc --silent ava --watch",
     "cov": "run-s build test:unit cov:html cov:lcov && open-cli coverage/index.html",
     "cov:html": "nyc report --reporter=html",
     "cov:lcov": "nyc report --reporter=lcov",
@@ -61,10 +59,12 @@
   "devDependencies": {
     "@ava/typescript": "^1.1.1",
     "@istanbuljs/nyc-config-typescript": "^1.0.1",
+    "@types/chai": "^4.3.1",
+    "@types/mocha": "^9.1.1",
     "@types/node": "^17.0.2",
     "@typescript-eslint/eslint-plugin": "^4.0.1",
     "@typescript-eslint/parser": "^4.0.1",
-    "ava": "^3.12.1",
+    "chai": "^4.3.6",
     "codecov": "^3.5.0",
     "cspell": "^4.1.0",
     "cz-conventional-changelog": "^3.3.0",
@@ -74,6 +74,7 @@
     "eslint-plugin-functional": "^3.0.2",
     "eslint-plugin-import": "^2.22.0",
     "gh-pages": "^3.1.0",
+    "mocha": "^10.0.0",
     "npm-run-all": "^4.1.5",
     "nyc": "^15.1.0",
     "open-cli": "^6.0.1",
@@ -92,18 +93,6 @@
     "LICENSE",
     "README.md"
   ],
-  "ava": {
-    "failFast": true,
-    "timeout": "60s",
-    "typescript": {
-      "rewritePaths": {
-        "src/": "build/main/"
-      }
-    },
-    "files": [
-      "!build/module/**"
-    ]
-  },
   "config": {
     "commitizen": {
       "path": "cz-conventional-changelog"

Diff for: src/bin/cli-bin.ts

@@ -16,13 +16,14 @@ function CLIErrorHandler(e: Error) {
 
 async function main() {
   program
-    .version("0.2.28")
+    .version("0.3.0")
     .description('The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.')
 
   program
     .command('scan <source>')
     .description('Scan a folder/file')
     .option('-w, --wfp', 'Scan a .wfp file instead of a folder')
+    .option('-H, --hpsm', 'Scan using winnowing high precision matching')
     .option('-c, --concurrency <number>', 'Number of concurrent connections to use while scanning (optional -default 10)')
     .option('-n, --ignore <ignore>',  'Ignore components specified in the SBOM file')
     .option('-f, --filter <path>', 'Loads an user defined filter (optional)')
@@ -51,6 +52,7 @@ async function main() {
     program
     .command('fingerprint <source>')
     .description('Generates fingerprints for a folder/file')
+    .option('-H, --hpsm', 'Scan using winnowing high precision matching')
     .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
     .option('-p, --block-size <size>', 'Maximum size in Kb for each fingerprint block (optional - default 64Kb)')
     .action((source, options) => {fingerprintHandler(source, options).catch((e) => {CLIErrorHandler(e)})})

Diff for: src/commands/fingerprint.ts

@@ -1,11 +1,14 @@
-import { isFolder } from "./helpers";
-import { ScannerEvents, WfpCalculator } from "..";
-import { Tree } from "../lib/tree/Tree";
-import { FilterList } from "../lib/filters/filtering";
-import { FingerprintPackage } from "../lib/scanner/WfpProvider/FingerprintPackage";
+import { isFolder } from './helpers';
+import { ScannerEvents, WfpCalculator, WinnowingMode } from '..';
+import { Tree } from '../lib/tree/Tree';
+import { FilterList } from '../lib/filters/filtering';
+import {
+  FingerprintPackage
+} from '../lib/scanner/WfpProvider/FingerprintPackage';
 import fs from 'fs';
-import { defaultFilter } from "../lib/filters/defaultFilter";
+import { defaultFilter } from '../lib/filters/defaultFilter';
 import cliProgress from 'cli-progress';
+import { IWfpProviderInput } from '../lib/scanner/WfpProvider/WfpProvider';
 
 
 export async function fingerprintHandler(rootPath: string, options: any): Promise<void> {
@@ -15,14 +18,19 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
   const pathIsFolder = await isFolder(rootPath);
   const wfpCalculator = new WfpCalculator();
 
-  const tree = new Tree(rootPath);
-  const filter = new FilterList('');
-  filter.load(defaultFilter as FilterList);
+  let filesToFingerprint: string[] = [];
+  if (pathIsFolder) {
+    const tree = new Tree(rootPath);
+    const filter = new FilterList('');
+    filter.load(defaultFilter as FilterList);
 
-  tree.loadFilter(filter);
-  tree.buildTree();
+    tree.loadFilter(filter);
+    tree.buildTree();
+    filesToFingerprint = tree.getFileList();
+  } else {
+    filesToFingerprint.push(rootPath)
+  }
 
-  const filesToFingerprint = tree.getFileList();
 
   const optBar1 = { format: 'Fingerprinting Progress: [{bar}] {percentage}% | Fingerprinted {value} files of {total}' };
   const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
@@ -48,8 +56,9 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
     }
   });
 
-
-  wfpCalculator.start({fileList: filesToFingerprint, folderRoot: rootPath});
+  const wfpInput: IWfpProviderInput = {fileList: filesToFingerprint, folderRoot: rootPath}
+  if(options.hpsm) wfpInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM;
+  wfpCalculator.start(wfpInput);
 
 
 }

Diff for: src/commands/scan.ts

@@ -1,10 +1,17 @@
 import { Scanner } from '../lib/scanner/Scanner';
-import { SbomMode, ScannerEvents, ScannerInput } from '../lib/scanner/ScannerTypes';
+import {
+  SbomMode,
+  ScannerEvents,
+  ScannerInput,
+  WinnowingMode
+} from '../lib/scanner/ScannerTypes';
 import { ScannerCfg } from '../lib/scanner/ScannerCfg';
 import { Tree } from '../lib/tree/Tree';
 
 import cliProgress from 'cli-progress';
-import { DispatcherResponse } from '../lib/scanner/Dispatcher/DispatcherResponse';
+import {
+  DispatcherResponse
+} from '../lib/scanner/Dispatcher/DispatcherResponse';
 import { defaultFilter } from '../lib/filters/defaultFilter';
 import { FilterList } from '../lib/filters/filtering';
 
@@ -80,6 +87,7 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
   });
 
   if (options.wfp) scannerInput.wfpPath = rootPath;
+  if (options.hpsm) scannerInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM
 
   if (options.ignore) {
     scannerInput.sbom = fs.readFileSync(options.ignore, 'utf-8');

Diff for: src/lib/dependencies/DependencyScanner.ts

@@ -26,9 +26,10 @@ export class DependencyScanner {
     const grpcResponse = await this.grpcDependencyService.get(request);
     const response = grpcResponse.toObject();
 
-
     // Extract scope from localDependencies and add it to response
-    this.mergeScopeField(localDependencies, response);
+    // Also adds the requirements field from localDependency to the response if the server didn't
+    // replay back a version
+    this.repairOutput(localDependencies, response);
     return response;
   }
 
@@ -71,27 +72,34 @@ export class DependencyScanner {
     }
   }
 
-  private mergeScopeField(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject
-    ): IDependencyResponse {
-
-    const scopeHashMap = {};
+  private repairOutput(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject) {
 
+    // Create a map with key = [filename + purl] and the value is an object containing:
+    // * The scope of the local dependency
+    // * The requirement of the local dependency
+    // Later this map is used to add information in the server response
+    const localDependencyInfo = {};
     for (const file of localdependency.files) {
       const filename = file.file
-      for (const dependency of file.purls) {
-        if (dependency?.scope) scopeHashMap[filename + dependency.purl] = dependency.scope;
+      for (const localDependency of file.purls) {
+        const localInfo = {}
+        if (localDependency?.scope) localInfo['scope'] = localDependency.scope
+        if(localDependency?.requirement) localInfo['requirement'] = localDependency.requirement
+        localDependencyInfo[filename + localDependency.purl] = localInfo;
       }
     }
 
     for (const file of serverResponse.filesList) {
       const filename = file.file
       for (const dependency of file.dependenciesList) {
-        const scope = scopeHashMap[filename + dependency.purl];
-        if (scope) dependency['scope'] = scope;
+        const localDependencyData = localDependencyInfo[filename + dependency.purl];
+        if (localDependencyData?.scope) dependency['scope'] = localDependencyData.scope;
+        if (localDependencyData?.requirement && dependency.version == "") {
+          dependency.version = localDependencyData.requirement;
+        }
       }
     }
-
-    return serverResponse;
   }
 
+
 }

Diff for: src/lib/dependencies/LocalDependency/LocalDependency.ts

@@ -3,9 +3,13 @@ import fs from 'fs';
 import { ParserFuncType, ILocalDependencies } from "./DependencyTypes";
 import { requirementsParser } from "./parsers/pyParser";
 import { pomParser } from "./parsers/mavenParser";
-import { packagelockParser, packageParser } from "./parsers/npmParser";
+import {
+  packagelockParser,
+  packageParser,
+  yarnLockParser
+} from './parsers/npmParser';
 import { gemfilelockParser, gemfileParser } from "./parsers/rubyParser";
-import { goModParser } from './parsers/golangParser';
+import { goModParser, goSumParser } from './parsers/golangParser';
 
 export class LocalDependencies {
 
@@ -24,6 +28,8 @@ export class LocalDependencies {
         'Gemfile': gemfileParser,
         'Gemfile.lock': gemfilelockParser,
         'go.mod': goModParser,
+        'go.sum': goSumParser,
+        'yarn.lock': yarnLockParser
       };
   }
 

Diff for: src/lib/dependencies/LocalDependency/parsers/golangParser.ts

@@ -3,15 +3,6 @@ import { ILocalDependency } from "../DependencyTypes";
 import { PackageURL } from "packageurl-js";
 import path from "path";
 
-function parseModule (str: string) {
-  const res = /(?<type>[^\s]+)(?:\s)+(?<ns_name>[^\s]+)\s?(?<version>(.*))/.exec(str);
-  return {
-    type: res.groups.type,
-    ns_name: res.groups.ns_name,
-    version: res.groups.version
-  };
-}
-
 function parseDepLink (str: string) {
   const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))/.exec(str);
   return {
@@ -20,6 +11,16 @@ function parseDepLink (str: string) {
   };
 }
 
+function getDepDataGoModFromLine(line: string) {
+  const {ns_name, version} = parseDepLink(line);
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
 // Removes comments and spaces
 function preprocessLine(line: string) {
     if (line.includes("//"))
@@ -45,7 +46,6 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
   const lines =	fileContent.split('\n');
 
 	const require = [];
-	const exclude = [];
 
   for (let num = 0 ; num < lines.length ; num+=1) {
 
@@ -57,11 +57,7 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
       line = preprocessLine(lines[num]);
       while (num < lines.length && line!==')') {
 
-        const {ns_name, version} = parseDepLink(line);
-
-        const index = ns_name.lastIndexOf('/');
-        const namespace = ns_name.substring(0, index);
-        const name = ns_name.substring(index + 1);
+        const {namespace, name, version} = getDepDataGoModFromLine(line)
 
         const purlString = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString();
         results.purls.push({purl: purlString});
@@ -76,3 +72,59 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
 
   return results;
 }
+
+
+
+
+
+function parseGoSumDepLink (str: string) {
+  const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))\s+h1:(?<checksum>(.*))/.exec(str);
+  return {
+    ns_name: res?.groups?.ns_name,
+    version: res?.groups?.version,
+    checksum: res?.groups?.checksum
+  };
+}
+
+function getDepDataGoSumFromLine(line: string) {
+  const {ns_name, version} = parseGoSumDepLink(line);
+
+  if (!ns_name) return {};
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
+// See reference on: https://go.dev/ref/mod#go-mod-file
+export function goSumParser(fileContent: string, filePath: string): ILocalDependency {
+
+  // If the file is not a go.mod manifest file, return an empty results
+  const results: ILocalDependency = { file: filePath, purls: [] };
+  if (path.basename(filePath) != 'go.sum')
+    return results;
+
+
+  const lines = fileContent.split('\n');
+  for (let num = 0; num < lines.length; num += 1) {
+
+    let line = preprocessLine(lines[num]);  //Deletes coments
+    if(!line) continue
+
+    line = line.replace('/go.mod', '')
+    const {namespace, name, version} = getDepDataGoSumFromLine(line)
+
+    if (!name) continue
+
+    //const purlString = new PackageURL(PURL_TYPE, namespace, name, undefined, undefined, undefined).toString();
+    const purlString = `pkg:${PURL_TYPE}/${namespace}/${name}`
+    results.purls.push({purl: purlString, requirement: version})
+  }
+
+  return results;
+
+
+}
+