Merge pull request #205 from luzhongyang/main

fix sca url bug

Author: QuantYork

AgentServer/settings.py

@@ -46,6 +46,7 @@
     'rest_framework.authtoken',
     'dongtai',
     'apiserver',
+    'sca',
     'drf_spectacular'
 ]
 
@@ -167,6 +168,9 @@
 ENGINE_URL = config.get("engine", "url")
 HEALTH_ENGINE_URL = urljoin(ENGINE_URL, "/api/engine/health")
 BASE_ENGINE_URL = config.get("engine", "url") + '/api/engine/run?method_pool_id={id}'
+SCA_ENGINE_URL = config.get("engine","url") + '/api/engine/sca?agent_id={agent_id}' \
+                            + '&package_path={package_path}&package_signature={package_signature}' \
+                            + '&package_name={package_name}&package_algorithm={package_algorithm}'
 REPLAY_ENGINE_URL = config.get("engine", "url") + '/api/engine/run?method_pool_id={id}&model=replay'
 
 LOGGING = {
@@ -194,7 +198,7 @@
             'backupCount': 5,
             'maxBytes': 1024 * 1024 * 10,
             'formatter': 'verbose',
-            'encoding':'utf8',
+            'encoding': 'utf8',
         },
     },
     'loggers': {
@@ -231,9 +235,5 @@
 IGNORE = 4
 SOLVED = 5
 
-# SCA_URL
-SCA_URL = config.get("sca", 'url')
-
 if os.getenv('environment', None) == 'DEV' or os.getenv('PYTHONAGENT', None) == 'TRUE':
-    MIDDLEWARE.append('dongtai_agent_python.middlewares.django_middleware.FireMiddleware')
-
+    MIDDLEWARE.append('dongtai_agent_python.middlewares.django_middleware.FireMiddleware')

AgentServer/urls.py

@@ -21,6 +21,7 @@
 
 urlpatterns = [
     path('api/v1/', include('apiserver.urls'), name='OpenAPI'),
+    path('sca/v1/', include('sca.urls'), name='ScaAPI'),
 ]
 
 if settings.DEBUG:

apiserver/report/handler/sca_handler.py

@@ -35,79 +35,28 @@ def parse(self):
         self.package_name = self.detail.get('packageName')
         self.package_algorithm = self.detail.get('packageAlgorithm')
 
+    @staticmethod
+    def send_to_engine(agent_id, package_path, package_signature, package_name, package_algorithm):
+        try:
+            logger.info(f'[+] send sca package [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm}] to engine')
+            requests.get(
+                url=settings.SCA_ENGINE_URL.format(
+                    agent_id=agent_id,
+                    package_path=package_path,
+                    package_signature=package_signature,
+                    package_name=package_name,
+                    package_algorithm=package_algorithm,
+                )
+            )
+        except Exception as e:
+            logger.info(f'[-] Failure: sca package [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm}], Error: {e}')
+
     def save(self):
         if all([self.agent_id, self.package_path, self.package_name]) is False:
             logger.warning(_("Data is incomplete, data: {}").format(json.dumps(self.report)))
         else:
-            search_query = ""
-            if self.agent.language == "JAVA":
-                version = self.package_name.split('/')[-1].replace('.jar', '').split('-')[-1]
-                search_query = "hash=" + self.package_signature
-            elif self.agent.language == "PYTHON":
-                # @todo agent上报版本 or 捕获全量pip库
-                version = self.package_name.split('/')[-1].split('-')[-1]
-                name = self.package_name.replace("-" + version, "")
-                search_query = "ecosystem={}&name={}&version={}".format("PyPI", name, version)
-            if search_query != "":
-                package_name = self.package_name
-                level = 'info'
-                try:
-                    url = settings.SCA_URL + "/api/package_vul/?" + search_query
-                    resp = requests.get(url=url)
-                    resp = json.loads(resp.content)
-                    maven_model = resp.get("data", {}).get("package", {})
-                    if maven_model is None:
-                        maven_model = {}
-                    vul_list = resp.get("data", {}).get("vul_list", {})
-                    package_name = maven_model.get('aql', self.package_name)
-                    version = maven_model.get('version', version)
-                    vul_count = len(vul_list)
-                    levels = []
-                    for vul in vul_list:
-                        _level = vul.get("vul_package", {}).get("severity", "none")
-                        if _level and _level not in levels:
-                            levels.append(_level)
-                    if len(levels) > 0:
-                        levels = [_['vul_level'] for _ in levels]
-                        if 'high' in levels:
-                            level = 'high'
-                        elif 'high' in levels:
-                            level = 'high'
-                        elif 'medium' in levels:
-                            level = 'medium'
-                        elif 'low' in levels:
-                            level = 'low'
-                        else:
-                            level = 'info'
-
-                except Exception as e:
-                    logger.info("get package_vul failed:{}".format(e))
-
-                try:
-                    level = IastVulLevel.objects.get(name=level)
-                    current_version_agents = self.get_project_agents(self.agent)
-                    asset_count = 0
-                    if current_version_agents:
-                        asset_count = Asset.objects.values("id").filter(signature_value=self.package_signature,
-                                                                        agent__in=current_version_agents).count()
-
-                    if asset_count == 0:
-                        Asset.objects.create(
-                            package_name=package_name,
-                            package_path=self.package_path,
-                            signature_algorithm=self.package_algorithm,
-                            signature_value=self.package_signature,
-                            dt=time.time(),
-                            version=version,
-                            level=level,
-                            vul_count=vul_count,
-                            agent=self.agent
-                        )
-                    project = IastProject.objects.filter(pk=self.agent.bind_project_id).first()
-                    if project:
-                        project.update_latest()
-                except Exception as e:
-                    logger.error(_('SCA data resolution failed, reasons: {}').format(e))
+            # post to dongtai engine async deal
+            ScaHandler.send_to_engine(self.agent_id, self.package_path, self.package_signature, self.package_name, self.package_algorithm)
 
 @ReportHandler.register(const.REPORT_SCA + 1)
 class ScaBulkHandler(ScaHandler):

conf/config.ini.example

@@ -22,6 +22,3 @@ from_addr = from_add
 ssl = ssl
 cc_addr = cc_addr
 
-[sca]
-url = http://52.80.75.225:8000
-

sca/__init__.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ScaConfig(AppConfig):
+    name = 'sca'

sca/apps.py

@@ -0,0 +1,82 @@
+from django.db import models
+
+# Create your models here.
+
+from django.db import models
+
+
+class Package(models.Model):
+    aql = models.CharField(max_length=255, blank=True, null=True)
+    hash = models.CharField(max_length=255, blank=True, null=True)
+    ecosystem = models.CharField(max_length=50, blank=True, null=True)
+    name = models.CharField(max_length=255, blank=True, null=True)
+    version = models.CharField(max_length=255, blank=True, null=True)
+    license = models.CharField(max_length=50, blank=True, null=True)
+
+    created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True)
+    updated_at = models.DateTimeField(auto_now=True, blank=True, null=True)
+
+    class Meta:
+        db_table = 'sca2_package'
+
+
+class Vul(models.Model):
+    id = models.CharField(primary_key=True, max_length=50)
+    summary = models.CharField(max_length=255, blank=True, null=True)
+    details = models.TextField(blank=True, null=True)
+    aliases = models.JSONField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
+    published = models.DateTimeField(blank=True, null=True)
+    withdrawn = models.DateTimeField(blank=True, null=True)
+    references = models.JSONField(null=True)
+
+    created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True)
+    updated_at = models.DateTimeField(auto_now=True, blank=True, null=True)
+
+    class Meta:
+        db_table = 'sca2_vul'
+
+
+class VulPackage(models.Model):
+    vul_id = models.CharField(max_length=50, blank=True, null=True)
+    ecosystem = models.CharField(max_length=255, blank=True, null=True)
+    name = models.CharField(max_length=255, blank=True, null=True)
+    cwe_ids = models.JSONField(blank=True, null=True)
+    ghsa = models.CharField(max_length=255, blank=True, null=True)
+    cvss_vector = models.CharField(max_length=255, blank=True, null=True)
+    cvss_score = models.FloatField(default=0, blank=True, null=True)
+    source = models.CharField(max_length=255, blank=True, null=True)
+    severity = models.CharField(max_length=50, blank=True, null=True)
+    created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True)
+    updated_at = models.DateTimeField(auto_now=True, blank=True, null=True)
+
+    class Meta:
+        db_table = 'sca2_vul_package'
+
+
+class VulPackageRange(models.Model):
+    vul_package_id = models.IntegerField(max_length=11, blank=True, null=True)
+    ecosystem = models.CharField(max_length=255, blank=True, null=True)
+    name = models.CharField(max_length=255, blank=True, null=True)
+    type = models.CharField(max_length=50, blank=True, null=True)
+    introduced = models.CharField(max_length=50, blank=True, null=True)
+    fixed = models.CharField(max_length=50, blank=True, null=True)
+
+    created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True)
+    updated_at = models.DateTimeField(auto_now=True, blank=True, null=True)
+
+    class Meta:
+        db_table = 'sca2_vul_package_range'
+
+
+class VulPackageVersion(models.Model):
+    vul_package_id = models.IntegerField(max_length=11, blank=True, null=True)
+    ecosystem = models.CharField(max_length=255, blank=True, null=True)
+    name = models.CharField(max_length=255, blank=True, null=True)
+    version = models.CharField(max_length=255, blank=True, null=True)
+
+    created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True)
+    updated_at = models.DateTimeField(auto_now=True, blank=True, null=True)
+
+    class Meta:
+        db_table = 'sca2_vul_package_version'

sca/models.py

@@ -0,0 +1,11 @@
+from django.urls import include, path
+from sca.views.package import PackageList
+from sca.views.package_vul import OnePackageVulList
+from rest_framework import routers
+router = routers.DefaultRouter()
+
+urlpatterns = [
+    path('package/', PackageList.as_view()),
+    path('package_vul/', OnePackageVulList.as_view()),
+]
+

sca/urls.py

@@ -0,0 +1,40 @@
+from sca.models import Package
+from django.http import JsonResponse
+from rest_framework import views
+from django.core.paginator import Paginator
+from django.forms.models import model_to_dict
+
+
+class PackageList(views.APIView):
+
+    def get(self, request):
+        filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version']
+        _filter = Package.objects.filter().order_by("-updated_at")
+        kwargs = {}
+        for filter_field in filter_fields:
+            _val = request.GET.get(filter_field, "")
+            if _val != "":
+                kwargs[filter_field] = request.GET.get(filter_field, "")
+        _filter = _filter.filter(**kwargs)
+
+        page = int(request.GET.get("page", 1))
+        page_size = int(request.GET.get("page_size", 5))
+
+        pageinfo = Paginator(_filter, per_page=page_size)
+        result = {
+            'data': [],
+            'msg': 'success',
+            'page': {
+                'alltotal': pageinfo.count,
+                'num_pages': pageinfo.num_pages,
+                'page_size': pageinfo.per_page,
+            },
+            'status': 201
+        }
+        if page == 0 or page <= pageinfo.num_pages:
+            rows = pageinfo.page(page).object_list
+
+            for row in rows:
+                result['data'].append(model_to_dict(row))
+
+        return JsonResponse(result)

sca/views/package.py

@@ -0,0 +1,84 @@
+from sca.models import Package, VulPackageVersion, VulPackage, VulPackageRange, Vul
+from django.http import JsonResponse
+from rest_framework import views
+from django.forms.models import model_to_dict
+
+class OnePackageVulList(views.APIView):
+
+    # 查找单个漏洞下，所有的修复的高版本
+    def find_fixed_versions(self, vul_package_id, ecosystem, name, version):
+        vul_package_ranges = VulPackageRange.objects.filter(
+            vul_package_id=vul_package_id,
+            ecosystem=ecosystem, name=name,
+            type__in=['ECOSYSTEM', 'SEMVER'],
+            # introduced__lte=version,
+            fixed__gte=version
+        ).all()
+        fixed_versions = []
+        for vul_package_range in vul_package_ranges:
+            fixed_versions.append(vul_package_range.fixed)
+        return fixed_versions
+
+    def get(self, request):
+        filter_fields = ['hash', 'aql', 'ecosystem', 'name', 'version']
+        _filter = Package.objects.filter()
+        kwargs = {}
+        for filter_field in filter_fields:
+            _val = request.GET.get(filter_field, "")
+            if _val != "":
+                kwargs[filter_field] = request.GET.get(filter_field, "")
+
+        ecosystem = request.GET.get('ecosystem', '')
+        name = request.GET.get('name', '')
+        version = request.GET.get('version', '')
+
+        package = _filter.filter(**kwargs).first()
+        print(package)
+        if package is not None:
+            ecosystem = package.ecosystem
+            name = package.name
+            version = package.version
+
+        vul_list = []
+        vul_package_ids = []
+        vul_package_ranges = VulPackageRange.objects.filter(
+            ecosystem=ecosystem, name=name, type="SEMVER",
+            introduced__gte=version, fixed__lte=version
+        ).all()[0:1000]
+
+        for vul_package_range in vul_package_ranges:
+            vul_package_ids.append(vul_package_range.vul_package_id)
+
+        vul_package_versions = VulPackageVersion.objects.filter(
+            ecosystem=ecosystem, name=name, version=version
+        ).all()[0:1000]
+        for vul_package_version in vul_package_versions:
+            vul_package_ids.append(vul_package_version.vul_package_id)
+
+        for vul_package_id in vul_package_ids:
+            vul_package = VulPackage.objects.get(pk=vul_package_id)
+            vul = Vul.objects.get(pk=vul_package.vul_id)
+            vul_list.append(
+                {
+                    "vul": model_to_dict(vul),
+                    "vul_package": model_to_dict(vul_package),
+                    "fixed_versions": self.find_fixed_versions(
+                        vul_package_id,
+                        ecosystem,
+                        name,
+                        version
+                    )
+                }
+            )
+        if package is not None:
+            package = model_to_dict(package)
+
+        result = {
+            'data': {
+                "vul_list": vul_list,
+                "package": package,
+            },
+            'msg': 'success',
+            'status': 201
+        }
+        return JsonResponse(result)