Merge pull request #633 from yeti-platform/1.6

1.6

Author: sebdraven

.gitignore

@@ -62,3 +62,11 @@ node_modules
 # Vagrant
 .vagrant/
 vagrant
+
+#IDE
+.idea
+
+#Patch
+*.patch
+#venv
+venv

Pipfile.lock

@@ -19,7 +19,7 @@
         <td class="icon-cell"><a href="{{ url_for('frontend.InvestigationView:graph_node', id=observable['id'], klass='observable')}}"><i class="flaticon-network38"></i></a></td>
         <td><a href="{{ url_for('frontend.ObservableView:get', id=observable['id'])}}">{{observable['value']}}</a></td>
         <td>{{macros.display_tags(observable['tags'])}}</td>
-        <td>{{ observable['context']|join(', ', attribute='source')}}</td>
+        <td>{{ observable['context']|unique(attribute='source')|join(', ', attribute='source')}}</td>
         <td>{{macros.display_datetime(observable['created'])}}</td>
         <td>{{ observable['sources']|join(', ')}}</td>
       </tr>

core/web/api/templates/observable_api.html

@@ -10,7 +10,7 @@
       <td>{{link.description}}</td>
       <td>{{ node.family.name }}</td>
       <td>{{ macros.display_tags(node.aliases)}}</td>
-      <td>{{node.context|join(', ', attribute='source')}}</td>
+      <td>{{node.context|unique(attribute='source')|join(', ', attribute='source')}}</td>
     </tr>
     {% endfor %}
   {%endfor%}

core/web/frontend/templates/macros/malware.html

@@ -32,7 +32,7 @@ <h1 class="panel-title">Observables present in database</h1>
         <table class="table table-condensed">
           <tr><th>Value</th><th>Tags</th><th>Context</th><th>Creation date</th></tr>
           {% for obs in data['known']%}
-          <tr><td><a href="{{url_for("frontend.ObservableView:get", id=obs["id"])}}">{{obs['value']}}</a></td><td>{{macros.display_tags(obs['tags'])}}</td><td>{{obs['context']|join(', ', attribute='source')}}</td><td>{{macros.display_datetime(obs['created'])}}</td></tr>
+          <tr><td><a href="{{url_for("frontend.ObservableView:get", id=obs["id"])}}">{{obs['value']}}</a></td><td>{{macros.display_tags(obs['tags'])}}</td><td>{{obs['context']|unique(attribute='source')|join(', ', attribute='source')}}</td><td>{{macros.display_datetime(obs['created'])}}</td></tr>
           {% endfor %}
         </table>
         {% endif %}
@@ -41,7 +41,7 @@ <h5>Interesting neighbors</h5>
 				<table class="table table-condensed">
           <tr><th>Value</th><th>Tags</th><th>Context</th><th>Creation date</th></tr>
           {% for link, obs in data["neighbors"]%}
-          <tr><td><a href="{{url_for("frontend.ObservableView:get", id=obs["id"])}}">{{obs["value"]}}</a> <span class="small">({% if obs["value"] == link["dst"]%}{{link["src"]}}{%else%}{{link["dst"]}}{%endif%}{%if link["description"]%} - {{link["description"]}}{%endif%})</span></td><td>{{macros.display_tags(obs["tags"])}}</td><td>{{obs["context"]|join(", ", attribute="source")}}</td><td>{{macros.display_datetime(obs["created"])}}</td></tr>
+          <tr><td><a href="{{url_for("frontend.ObservableView:get", id=obs["id"])}}">{{obs["value"]}}</a> <span class="small">({% if obs["value"] == link["dst"]%}{{link["src"]}}{%else%}{{link["dst"]}}{%endif%}{%if link["description"]%} - {{link["description"]}}{%endif%})</span></td><td>{{macros.display_tags(obs["tags"])}}</td><td>{{obs["context"]|unique(attribute="source")|join(", ", attribute="source")}}</td><td>{{macros.display_datetime(obs["created"])}}</td></tr>
           {% endfor %}
         </table>
         {% endif %}

core/web/frontend/templates/observable/search_results.html

@@ -79,7 +79,7 @@ <h3 class="panel-title" id="yeti-node-name" data-id="{{observable.id}}" data-cla
                     <td><a href="{{ url_for("frontend.ObservableView:get", id=node.id) }}">{{node.value}}</a></td>
                     <td>{{link.description}}</td>
                     <td>{{ macros.display_tags(node.tags)}}</td>
-                    <td>{{node.context|join(', ', attribute='source')}}</td>
+                    <td>{{node.context|unique(attribute='source')|join(', ', attribute='source')}}</td>
                   </tr>
                   {% endfor %}
                 {%endfor%}

core/web/frontend/templates/observable/single.html

@@ -5,30 +5,49 @@ Installation
 
 Installing Yeti is pretty straightforward. This procedure was tested on Ubuntu 18.04, but YMMV.
 
-Install dependencies::
+Install the dependencies needed to add new repositories::
 
-  $ sudo apt-get install build-essential git python-dev mongodb redis-server libxml2-dev libxslt-dev zlib1g-dev python-virtualenv wkhtmltopdf
+  $ sudo apt update && sudo apt install dirmngr gnupg wget curl apt-transport-https
 
 Install Yarn::
 
   $ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
   $ echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
-  $ sudo apt-get update && sudo apt-get install yarn
+  $ sudo apt update && sudo apt install yarn
+
+Install mongodb-org::
+  
+  $ wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -
+  $ echo "deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list 
+  $ sudo apt update
 
-Download Yeti:
+Install dependencies::
 
+  $ sudo apt install build-essential git python-dev mongodb-org mongodb-org-shell mongodb-org-server mongodb-org-mongos redis-server libcurl4 libxml2-dev libxslt-dev zlib1g-dev python-virtualenv wkhtmltopdf python-pip python3-pip
+
+Return at home and download Yeti:
+  
+  $ cd
   $ git clone https://github.com/yeti-platform/yeti.git
 
 Activate virtualenv if you want to, then install requirements::
 
   $ cd yeti
-  $ [sudo] pip install -r requirements.txt
+  $ sudo -H pip3 install -r requirements.txt
   $ yarn install
 
-Create the logging directory::
+Create user and the logging directory::
+  
+  $ sudo useradd -r -M -d /opt/yeti -s /usr/sbin/nologin yeti
+  $ sudo mkdir /var/log/yeti
+  $ sudo chown yeti /var/log/yeti
+
+Copy yeti on /opt filesystem, and configure::
 
-  $ [sudo] mkdir /var/log/yeti
-  $ [sudo] chown <user> /var/log/yeti
+  $ cd
+  $ sudo mv yeti /opt
+  $ sudo chown -R yeti:yeti /opt/yeti
+  $ sudo chmod +x /opt/yeti/yeti.py
 
 Quick & dirty
 -------------
@@ -45,7 +64,7 @@ This will only enable the web interface - if you want to use Feeds and Analytics
   $ celery -A core.config.celeryctl.celery_app worker --loglevel=ERROR -Q oneshot -n oneshot -c 2 --purge
   $ celery -A core.config.celeryctl beat -S core.scheduling.Scheduler --loglevel=ERROR
 
-Or, to bootstrap a production use instance of Yeti on Ubuntu 16.04 (without the Redis tweaks), everyone's favorite command::
+Or, to bootstrap a production use instance of Yeti on Ubuntu 18.04 (without the Redis tweaks), everyone's favorite command::
 
   $ curl https://raw.githubusercontent.com/yeti-platform/yeti/master/extras/ubuntu_bootstrap.sh | sudo /bin/bash
 
@@ -61,7 +80,7 @@ For production use, it may be better to daemonize Yeti and tweak redis for perfo
 
 Install ``nginx`` and ``uwsgi``::
 
-  $ sudo apt-get install nginx uwsgi
+  $ sudo apt install nginx uwsgi uwsgi-plugin-python3
 
 Optimize redis
 ^^^^^^^^^^^^^^
@@ -84,7 +103,7 @@ Add the following lines in ``/etc/rc.local``::
 Install systemd services
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-Copy all files in ``extras/systemd/*`` to ``/lib/systemd/system/``. If you'd
+Copy all files in ``extras/systemd/*`` to ``/etc/systemd/system/``. If you'd
 rather have the web content served through nginx (recommended for production),
 copy ``yeti_uwsgi.service``, otherwise you'll be fine with ``yeti_web.service``.
 
@@ -98,20 +117,52 @@ And start with::
 
 systemd protips::
 
-    $ sudo service yeti_web start|stop|restart
+    $ sudo service yeti_uwsgi start|stop|restart
     or
     $ sudo systemctl start|status|stop yeti_web
 
 To enable the systemd scripts once you've installed them::
 
-    sudo systemctl enable yeti_web
+    sudo systemctl enable yeti_uwsgi
+
+For install yeti with development webserver::
+    $ sudo systemctl enable mongod.service
+    $ sudo systemctl enable yeti_web.service
+    $ sudo systemctl enable yeti_oneshot.service
+    $ sudo systemctl enable yeti_feeds.service
+    $ sudo systemctl enable yeti_exports.service
+    $ sudo systemctl enable yeti_analytics.service
+    $ sudo systemctl enable yeti_beat.service
+    $ sudo systemctl start mongod.service
+    $ sudo systemctl start yeti_web.service
+    $ sudo systemctl start yeti_oneshot.service
+    $ sudo systemctl start yeti_feeds.service
+    $ sudo systemctl start yeti_exports.service
+    $ sudo systemctl start yeti_analytics.service
+    $ sudo systemctl start yeti_beat.service
+
+For install yeti with nginx reverse proxy::
+    $ sudo systemctl enable mongod.service
+    $ sudo systemctl enable yeti_uwsgi.service
+    $ sudo systemctl enable yeti_oneshot.service
+    $ sudo systemctl enable yeti_feeds.service
+    $ sudo systemctl enable yeti_exports.service
+    $ sudo systemctl enable yeti_analytics.service
+    $ sudo systemctl enable yeti_beat.service
+    $ sudo systemctl start mongod.service
+    $ sudo systemctl start yeti_uwsgi.service
+    $ sudo systemctl start yeti_oneshot.service
+    $ sudo systemctl start yeti_feeds.service
+    $ sudo systemctl start yeti_exports.service
+    $ sudo systemctl start yeti_analytics.service
+    $ sudo systemctl start yeti_beat.service
 
 If you're running nginx, add the following configuration to one of the nginx
 server directives::
 
   server {
       listen 80;
-      server_name yeti;
+      server_name yeti.domain.lan;
 
       location / {
           include uwsgi_params;

doc/installation.rst

@@ -43,6 +43,6 @@ This will start a Yeti service running on <http://localhost:5000/>
 
 To start a more performant container for web requests, run:
 
-    docker-compose -p yeti -f extras/docker/dev/docker-compose.yml run -p 8080:8080 yeti /docker-entrypoint.sh uwsgi-http
+    docker-compose -p yeti -f extras/docker/dev/docker-compose.yaml run -p 8080:8080 yeti /docker-entrypoint.sh uwsgi-http
 
 Then point your browser to <http://localhost:8080.>

extras/docker/README.md

@@ -6,7 +6,7 @@ After=mongodb.service redis.service
 Type=simple
 User=yeti
 WorkingDirectory=/opt/yeti
-ExecStart=/usr/local/bin/uwsgi --socket 127.0.0.1:8000 -w yeti --callable webapp --processes 4 --threads 2 --stats 127.0.0.1:9191
+ExecStart=/usr/bin/uwsgi --plugin python3 --socket 127.0.0.1:8000 -w yeti --callable webapp --processes 4 --threads 2 --stats 127.0.0.1:9191
 Restart=always
 KillSignal=SIGQUIT
 Type=notify

extras/systemd/yeti_uwsgi.service

@@ -1,5 +1,12 @@
-# Install dependencies
-export LC_ALL="en_US.UTF-8"
+#!/bin/bash
+
+set -e
+
+#Check execution with root
+
+if [[ "$EUID" -ne "0" ]]; then
+	echo "Run this script as root"
+fi
 
 if [ -f "/usr/bin/apt" ]; then
    APT="/usr/bin/apt"
@@ -8,43 +15,33 @@ else
 fi
 
 $APT update -y
-$APT install dirmngr
+$APT install -y dirmngr gnupg apt-transport-https curl wget
 
 curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 
-# https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4
-
-# https://wiki.ubuntu.com/Releases
-OS_CODENAME=`lsb_release -c --short`
-
-if [ $OS_CODENAME == "bionic" ] || [ $OS_CODENAME == "artful" ] || [ $OS_CODENAME == "zesty" ] || [ $OS_CODENAME == "yakkety" ] || [ $OS_CODENAME == "xenial" ]; then
-  echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list
-elif [ $OS_CODENAME == "wily" ] || [ $OS_CODENAME == "vivid" ] || [ $OS_CODENAME == "utopic" ] || [ $OS_CODENAME == "trusty" ]; then
-  echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list
-else
-  echo "[!] Installing on an unsupported or outdated version of Ubuntu, trying Trusty package for Mongo"
-  echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/4.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list
-fi
+wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
+echo "deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list
 
 $APT update -y
-$APT install build-essential git python-dev mongodb-org redis-server libcurl3 libxml2-dev libxslt-dev zlib1g-dev python-virtualenv python-pip nginx yarn -y
+$APT install -y build-essential git python-dev mongodb-org mongodb-org-server mongodb-org-mongos mongodb-org-shell redis-server libcurl4 libxml2-dev libxslt-dev zlib1g-dev python-virtualenv python-pip python3-pip nginx yarn uwsgi-plugin-python3
 
 # Clone project
 cd /opt
 git clone https://github.com/yeti-platform/yeti.git
 
 # Install requirements
 cd /opt/yeti
-pip install -r requirements.txt
-pip install uwsgi
+pip3 install -r requirements.txt
+pip3 install uwsgi
 yarn install
 
 # Configure services
-useradd yeti
+useradd -r -M -d /opt/yeti -s /usr/sbin/nologin yeti
 sudo mkdir /var/log/yeti
 sudo chown yeti /var/log/yeti
+chown -R yeti:yeti /opt/yeti
+chmod +x /opt/yeti/yeti.py
 cp extras/systemd/*.service /etc/systemd/system/
 systemctl enable mongod.service
 systemctl enable yeti_uwsgi.service
@@ -54,14 +51,12 @@ systemctl enable yeti_exports.service
 systemctl enable yeti_analytics.service
 systemctl enable yeti_beat.service
 systemctl daemon-reload
-chown -R yeti:yeti /opt/yeti
-chmod +x /opt/yeti/yeti.py
 
 # Configure nginx
 rm /etc/nginx/sites-enabled/default
 cp extras/nginx/yeti /etc/nginx/sites-available/
 ln -s /etc/nginx/sites-available/yeti /etc/nginx/sites-enabled/yeti
-service nginx restart
+systemctl reload nginx
 
 # Start services
 echo "[+] Starting services..."

extras/ubuntu_bootstrap.sh

@@ -7,7 +7,7 @@
     "bootstrap-tokenfield": "^0.12.0",
     "font-awesome": "^4.7.0",
     "front-markjs": "^8.9.1-1",
-    "handlebars": "^4.3.0",
+    "handlebars": "^4.5.3",
     "highlight.js": "^9.11.0",
     "inline-attachment": "^2.0.3",
     "jquery": "^3.5.0",

package.json

@@ -51,6 +51,8 @@ def analyze(self, item):
 
             context["country"] = country
             context["threat"] = category
+            context["reliability"] = item["number_1"]
+            context["risk"] = item["number_2"]
 
             ip.tag(category)
             ip.add_context(context)