Merge pull request #178 from mitre/release/2.0.0rc1

v2.0 release

Author: ptcNOP

.pre-commit-config.yaml

@@ -1,5 +1,6 @@
--   repo: [email protected]:pre-commit/pre-commit-hooks
-    sha: v1.2.0
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.4.0
     hooks:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
@@ -8,5 +9,5 @@
     -   id: mixed-line-ending
     -   id: flake8
         args:
-        - --ignore=E126,E127,E128,E402
+        - --ignore=E126,E127,E128,E402,F841
         - --max-line-length=120

.travis.yml

@@ -1,20 +1,34 @@
+dist: xenial
 sudo: required
 language: python
 python:
   - "2.7"
-  - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
 env:
   - MOZ_HEADLESS=1
 addons:
   firefox: latest
+  apt:
+    # Keep these in sync with the dependencies from the install.sh script
+    packages:
+      - build-essential
+      - curl
+      - dh-autoreconf
+      - gcc
+      - libffi-dev
+      - libfuzzy-dev
+      - python-dev
+      - git
+      - libssl-dev
+      - unzip
+      - libmagic-dev
 before_install:
   - npm install -g geckodriver
 install:
-  - yes "" | sudo -HE ./install.sh
-  - pip install -r requirements.txt
-  - python multiscanner.py init
+  - pip install -U pip setuptools
+  - pip install tox-travis pre-commit
 script:
-  - flake8 --exit-zero .
-  - pytest
+  - tox
+  - pre-commit run --all-files

MANIFEST.in

@@ -0,0 +1,13 @@
+graft multiscanner
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+recursive-exclude multiscanner\tests *
+
+recursive-include docs
+include README.md
+include LICENSE
+include AUTHORS
+include requirements.txt
+include requirements-dev.txt
+include requirements-test.txt
+include install.sh

README.md

@@ -36,7 +36,7 @@ Then run the following (substituting the actual file you want to scan for `<file
 $ git clone https://github.com/mitre/multiscanner.git
 $ cd multiscanner
 $ sudo -HE ./install.sh
-$ python multiscanner.py init
+$ multiscanner init
 ```
 
 This will generate a default configuration for you. Check `config.ini` to see what
@@ -45,13 +45,13 @@ modules are enabled. See [Configuration](http://multiscanner.readthedocs.io/en/l
 Now you can scan a file (substituting the actual file you want to scan for `<file>`):
 
 ``` bash
-$ python multiscanner.py <file>
+$ multiscanner <file>
 ```
 
 You can run the following to get a list of all of MultiScanner's command-line options:
 
 ``` bash
-$ python multiscanner.py --help
+$ multiscanner --help
 ```
 
 **Note**: If you are not on a RedHat or Debian based Linux distribution, instead of

__init__.py

@@ -1,7 +1,7 @@
 version: '3'
 services:
   elastic:
-    image: "docker.elastic.co/elasticsearch/elasticsearch:5.6.3"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:6.3.2"
     ports:
      - "9200:9200"
      - "9300:9300"
@@ -12,36 +12,16 @@ services:
   web:
     build:
       context: .
-      dockerfile: docker_utils/Dockerfile_web
-      # If you are behind a proxy, you must uncomment
-      # the next 3 lines
-      # args:
-      #  - http_proxy
-      #  - https_proxy
+      dockerfile: docker_utils/Dockerfile
+    command: multiscanner-web
     ports:
      - "127.0.0.1:8000:8000"
-    # If you are behind a proxy, you must set the
-    # proxy settings here (uncomment the next 4 lines)
-    # environment:
-    #  - "http_proxy=http://proxy.example:80"
-    #  - "https_proxy=http://proxy.example:80"
-    #  - "no_proxy=localhost,127.0.0.1"
   api:
     build:
       context: .
-      dockerfile: docker_utils/Dockerfile_api
-      # If you are behind a proxy, you must uncomment
-      # the next 3 lines
-      # args:
-      #  - http_proxy
-      #  - https_proxy
+      dockerfile: docker_utils/Dockerfile
+    command: /wait-for-it.sh elastic:9200 -- multiscanner-api
     ports:
      - "127.0.0.1:8080:8080"
-    # If you are behind a proxy, you must set the
-    # proxy settings here (uncomment the next 4 lines)
-    # environment:
-    #  - "http_proxy=http://proxy.example:80"
-    #  - "https_proxy=http://proxy.example:80"
-    #  - "no_proxy=localhost,127.0.0.1"
     depends_on:
      - elastic

docker-compose.yml

@@ -0,0 +1,90 @@
+FROM alpine
+MAINTAINER Patrick Copeland ptcnop
+
+ENV YARA_VERSION 3.8.1
+ENV YARA_PY_VERSION 3.8.1
+ENV SSDEEP ssdeep-2.13
+
+COPY requirements.txt /opt/multiscanner/
+
+RUN apk add --no-cache \
+        bash \
+        bison \
+        file \
+        jansson \
+        jpeg \
+        libffi \
+        python3 \
+        su-exec \
+        tini \
+        zip \
+        zlib \
+  && apk add --no-cache -t .build-deps \
+       autoconf \
+       automake \
+       build-base \
+       file-dev \
+       flex \
+       git \
+       jansson-dev \
+       jpeg-dev \
+       libc-dev \
+       libffi-dev \
+       libtool \
+       musl-dev \
+       postgresql-dev \
+       py3-pip \
+       python3-dev \
+       zlib-dev \
+  # ssdeep
+  && echo "Install ssdeep from source..." \
+  && cd /tmp \
+  && wget -O /tmp/$SSDEEP.tar.gz https://downloads.sourceforge.net/project/ssdeep/$SSDEEP/$SSDEEP.tar.gz \
+  && tar zxvf $SSDEEP.tar.gz \
+  && cd $SSDEEP \
+  && ./configure \
+  && make \
+  && make install \
+  # yara
+  && echo "Install Yara from source..." \
+  && cd /tmp/ \
+  && git clone --recursive --branch v$YARA_VERSION https://github.com/VirusTotal/yara.git \
+  && cd /tmp/yara \
+  && ./bootstrap.sh \
+  && sync \
+  && ./configure --with-crypto \
+  --enable-magic \
+  --enable-cuckoo \
+  --enable-dotnet \
+  && make \
+  && make install \
+  && echo "Install yara-python..." \
+  && cd /tmp/ \
+  && git clone --recursive --branch v$YARA_PY_VERSION https://github.com/VirusTotal/yara-python \
+  && cd yara-python \
+  && python3 setup.py build --dynamic-linking \
+  && python3 setup.py install \
+  && echo "Downloading yara signatures..." \
+  && git clone --depth 1 https://github.com/Yara-Rules/rules.git /opt/multiscanner/etc/yarasigs/Yara-Rules \
+  # install ms dependencies
+  && cd /opt/multiscanner \
+  && pip3 install --upgrade pip \
+  && pip3 install -r requirements.txt \
+  # clean up
+  && rm -rf /tmp/* \
+  && apk del --purge .build-deps
+
+COPY . /opt/multiscanner
+COPY ./docker_utils/*.ini /opt/multiscanner/
+COPY ./etc/pdf_config.json /opt/multiscanner/
+COPY ./etc/ember_model_2017.txt /opt/multiscanner/etc/ember/
+
+WORKDIR /opt/multiscanner
+
+RUN pip3 install .
+
+RUN wget https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -O /wait-for-it.sh \
+  && chmod +x /wait-for-it.sh
+
+# Run script
+CMD multiscanner

docker_utils/Dockerfile

@@ -23,3 +23,5 @@ host_string = localhost
 db_name = task_db
 username = multiscanner
 password = CHANGEME
+retry_time = 5
+retry_num = 20

docker_utils/Dockerfile_api

@@ -35,6 +35,12 @@ poll interval seconds = 5
 user agent = user_agent
 API key =
 
+[VFindScan]
+ENABLED = False
+vstk_home = /opt/vstk
+uad_cmdline = []
+vfind_cmdline = []
+
 [vtsearch]
 apikey = None
 ENABLED = False
@@ -84,6 +90,13 @@ Verify = False
 timeout = 360
 running timeout = 120
 
+[EndgameEmber]
+ENABLED = False
+path-to-model = opt/multiscanner/etc/ember/ember_model_2017.txt
+
+[MaliciousMacroBot]
+ENABLED = False
+
 [ExifToolsScan]
 cmdline = ['-t']
 path = C:\exiftool.exe
@@ -108,15 +121,32 @@ cmdline = ['-r:3']
 host = ('MultiScanner', 22, 'User')
 replacement path = X:\
 
+[UADScan]
+ENABLED = False
+vstk_home = /opt/vstk
+cmdline = []
+
+[entropy]
+ENABLED = True
+
+[fileextensions]
+ENABLED = True
+
 [flarefloss]
 ENABLED = False
 path = /opt/floss
 cmdline = ['--show-metainfo']
 
+[impfuzzy]
+ENABLED = True
+
 [libmagic]
 magicfile = None
 ENABLED = True
 
+[officemeta]
+ENABLED = True
+
 [pdfinfo]
 ENABLED = True
 fast = False

docker_utils/Dockerfile_web

@@ -1,12 +1,31 @@
-[File]
+[main]
+retry_time = 5
+retry_num = 20
+
+[BasicElasticSearchStorage]
 ENABLED = False
-path = report.json
-gzip = False
-files-per-line = 1
+host = elastic
+port = 9200
+index = multiscanner_reports
+doc_type = reports
 
 [ElasticSearchStorage]
 ENABLED = True
 host = elastic
 port = 9200
 index = multiscanner_reports
-doc_type = report
+metricbeat_enabled = True
+metricbeat_rollover_days = 7
+
+[File]
+ENABLED = False
+path = report.json
+gzip = False
+files-per-line = 1
+
+[MongoStorage]
+ENABLED = False
+host = localhost
+port = 27017
+database = multiscanner_reports
+collection = reports

docker_utils/api_config.ini

@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docker_utils/config.ini

@@ -11,4 +11,4 @@ Enabling analytics and advanced queries is the primary advantage of running seve
 
 Analytic development is currently ad hoc. Until interfaces are created to standardize development, the :ref:`analytics` section might prove useful - it contains development details of the **ssdeep** analytic.
 
-Here's the `ssdeep code <https://github.com/mitre/multiscanner/blob/feature-celery/analytics/ssdeep_analytics.py>`_ to use as a reference for how one might implement an analytic.
+Here's the `ssdeep code <https://github.com/mitre/multiscanner/blob/feature-celery/analytics/ssdeep_analytics.py>`_ to use as a reference for how one might implement an analytic.