Skip to content

Commit 9e8b76c

Browse files
sebdravensebdraven
authored
Merge pull request #716 from yeti-platform/1.8.5
1.8.5
2 parents d20cb96 + de8407e commit 9e8b76c

File tree

94 files changed

+2028
-301
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+2028
-301
lines changed

‎.travis.yml

+1-3
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ sudo: required
22

33
language: python
44
python:
5-
- "3.7"
5+
- "3.9"
66

77
services:
88
- redis
@@ -30,5 +30,3 @@ script:
3030
- ./yeti.py webserver &
3131
- sleep 5
3232
- curl -I -L http://127.0.0.1:5000
33-
- cd pyeti
34-
- nosetests

‎contrib/feeds/threattracking/threattracking.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,8 @@ def get_aliases(self, sheet_name, range_info):
109109
actor_names = res["values"]
110110
r_names = []
111111
for i, actor_aliases in enumerate(actor_names):
112-
while u"" in actor_aliases:
113-
actor_aliases.remove(u"")
112+
while "" in actor_aliases:
113+
actor_aliases.remove("")
114114
while "?" in actor_aliases:
115115
actor_aliases.remove("?")
116116
while "???" in actor_aliases:
@@ -150,8 +150,8 @@ def get_campaign(self, sheet_name, range_info):
150150
campaign_names = _["values"]
151151
r_names = []
152152
for i, campaigns in enumerate(campaign_names):
153-
while u"" in campaigns:
154-
campaigns.remove(u"")
153+
while "" in campaigns:
154+
campaigns.remove("")
155155
campaigns = list(set(campaigns))
156156
r_names.append(campaigns)
157157
return r_names
@@ -169,8 +169,8 @@ def get_tools(self, sheet_name, range_info):
169169
tools = tools[0].split(",")
170170
tools = [t.strip() for t in tools]
171171
tools = list(set(tools))
172-
while u"" in tools:
173-
tools.remove(u"")
172+
while "" in tools:
173+
tools.remove("")
174174
r_names.append(tools)
175175
return r_names
176176

‎core/auth/apache/views.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ def login():
1414
user = authenticate()
1515
if user:
1616
login_user(user)
17-
print "User logged in (web):", user
17+
print("User logged in (web):".format(user))
1818
return redirect(request.args.get("next", "/"))
1919
flash("Invalid credentials", "danger")
2020
abort(401)

‎core/config/config.py

+1
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,4 @@ def get(self, section, key, default=None):
6161
yeti_config.set_default_value("proxy", "http", None)
6262
yeti_config.set_default_value("proxy", "https", None)
6363
yeti_config.set_default_value("logging", "filename", "/var/log/yeti/user_activity.log")
64+
yeti_config.set_default_value("tag", "default_tag_expiration", 7776000)

‎core/exports/export.py

+27-10
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import traceback
88
import hashlib
99

10-
from mongoengine import ListField, StringField, Q, ReferenceField, PULL
10+
from mongoengine import ListField, StringField, Q, ReferenceField, PULL, BooleanField
1111
from jinja2 import Environment, FileSystemLoader
1212
from flask import url_for
1313
from mongoengine import DoesNotExist
@@ -95,6 +95,7 @@ class Export(ScheduleEntry):
9595
include_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
9696
exclude_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
9797
ignore_tags = ListField(ReferenceField(Tag, reverse_delete_rule=PULL))
98+
fresh_tags = BooleanField(default=True)
9899
output_dir = StringField(default="exports")
99100
acts_on = StringField(verbose_name="Acts on")
100101
template = ReferenceField(ExportTemplate)
@@ -115,15 +116,31 @@ def content_uri(self):
115116

116117
def execute(self):
117118
q_include = Q()
118-
for t in self.include_tags:
119-
q_include |= Q(tags__match={"name": t.name, "fresh": True})
120-
q_exclude = Q(tags__name__nin=[t.name for t in self.exclude_tags])
121-
q = (
122-
Q(tags__not__size=0, tags__match={"fresh": True})
123-
& q_include
124-
& q_exclude
125-
& Q(_cls="Observable.{}".format(self.acts_on))
126-
)
119+
q_exclude = Q()
120+
if self.fresh_tags: # including
121+
for t in self.include_tags:
122+
q_include |= Q(tags__match={"name": t.name, "fresh": True})
123+
else:
124+
q_include |= Q(tags__name__in=[t.name for t in self.include_tags])
125+
if self.fresh_tags: # excluding
126+
for t in self.exclude_tags:
127+
q_exclude |= Q(tags__match__ne={"name": t.name, "fresh": True})
128+
else:
129+
q_exclude |= Q(tags__name__nin=[t.name for t in self.exclude_tags])
130+
if self.fresh_tags:
131+
q = (
132+
Q(tags__not__size=0, tags__match={"fresh": True})
133+
& q_include
134+
& q_exclude
135+
& Q(_cls="Observable.{}".format(self.acts_on))
136+
)
137+
else:
138+
q = (
139+
Q(tags__not__size=0)
140+
& q_include
141+
& q_exclude
142+
& Q(_cls="Observable.{}".format(self.acts_on))
143+
)
127144

128145
return self.template.render(
129146
self.filter_ignore_tags(Observable.objects(q).no_cache()), self.output_file

‎core/feed.py

+13
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ def _choose(
198198
quotechar='"',
199199
quoting=True,
200200
skipinitialspace=True,
201+
index_col=False,
201202
)
202203

203204
elif header and not comment and not names:
@@ -210,7 +211,9 @@ def _choose(
210211
quotechar='"',
211212
quoting=True,
212213
skipinitialspace=True,
214+
index_col=False,
213215
)
216+
214217
elif header and comment and not names:
215218
df = pd.read_csv(
216219
StringIO(feed),
@@ -222,6 +225,7 @@ def _choose(
222225
quotechar='"',
223226
quoting=True,
224227
skipinitialspace=True,
228+
index_col=False,
225229
)
226230

227231
elif not header and comment and not names:
@@ -234,7 +238,9 @@ def _choose(
234238
quotechar='"',
235239
quoting=True,
236240
skipinitialspace=True,
241+
index_col=False,
237242
)
243+
238244
elif not header and not comment and not names:
239245
df = pd.read_csv(
240246
StringIO(feed),
@@ -244,6 +250,7 @@ def _choose(
244250
quotechar='"',
245251
quoting=True,
246252
skipinitialspace=True,
253+
index_col=False,
247254
)
248255
else:
249256

@@ -256,6 +263,7 @@ def _choose(
256263
quotechar='"',
257264
quoting=True,
258265
skipinitialspace=True,
266+
index_col=False,
259267
)
260268
elif not comment and names:
261269
df = pd.read_csv(
@@ -266,6 +274,7 @@ def _choose(
266274
quotechar='"',
267275
quoting=True,
268276
skipinitialspace=True,
277+
index_col=False,
269278
)
270279
elif header and not comment and not names:
271280
df = pd.read_csv(
@@ -275,6 +284,7 @@ def _choose(
275284
quotechar='"',
276285
quoting=True,
277286
skipinitialspace=True,
287+
index_col=False,
278288
)
279289
elif header and comment and not names:
280290
df = pd.read_csv(
@@ -285,6 +295,7 @@ def _choose(
285295
quotechar='"',
286296
quoting=True,
287297
skipinitialspace=True,
298+
index_col=False,
288299
)
289300

290301
elif not header and comment and not names:
@@ -295,6 +306,7 @@ def _choose(
295306
quotechar='"',
296307
quoting=True,
297308
skipinitialspace=True,
309+
index_col=False,
298310
)
299311
elif not header and not comment and not names:
300312
df = pd.read_csv(
@@ -303,6 +315,7 @@ def _choose(
303315
quotechar='"',
304316
quoting=True,
305317
skipinitialspace=True,
318+
index_col=False,
306319
)
307320

308321
return df

‎core/internals/migrations/wkhtmltopdf_3.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33

44
def migrate():
5-
print """
5+
print(
6+
"""
67
78
# Note
89
@@ -22,3 +23,4 @@ def migrate():
2223
$ brew install wkhtmltopdf
2324
2425
"""
26+
)

‎core/observables/__init__.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
from core.observables.tag import ObservableTag, Tag
44
from core.observables.observable import Observable
5-
from core.observables.ip import Ip, AutonomousSystem
5+
from core.observables.ip import Ip
6+
from core.observables.asn import AutonomousSystem
67
from core.observables.url import Url
78
from core.observables.hostname import Hostname
89
from core.observables.hash import Hash

‎core/observables/asn.py

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
from core.observables import Observable
2+
from mongoengine import IntField
3+
4+
5+
class AutonomousSystem(Observable):
6+
7+
"""Autonomous System observable"""
8+
9+
as_num = IntField(verbose_name="Autonomous System number")
10+
11+
DISPLAY_FIELDS = Observable.DISPLAY_FIELDS + [
12+
("as_num", "Autonomous System number"),
13+
]
14+
15+
def info(self):
16+
info = super(AutonomousSystem, self).info()
17+
info["as_num"] = (self.as_num,)
18+
return info
19+
20+
@staticmethod
21+
def check_type(txt):
22+
return True

‎core/observables/ip.py

-20
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,3 @@ def normalize(self):
5050
elif iptools.ipv6.validate_ip(self.value):
5151
self.value = iptools.ipv6.long2ip(iptools.ipv6.ip2long(self.value))
5252
self.version = 6
53-
54-
55-
class AutonomousSystem(Observable):
56-
57-
"""Autonomous System observable"""
58-
59-
as_num = IntField(verbose_name="Autonomous System number")
60-
61-
DISPLAY_FIELDS = Observable.DISPLAY_FIELDS + [
62-
("as_num", "Autonomous System number"),
63-
]
64-
65-
def info(self):
66-
info = super(AutonomousSystem, self).info()
67-
info["as_num"] = (self.as_num,)
68-
return info
69-
70-
@staticmethod
71-
def check_type(txt):
72-
return True

‎core/observables/tag.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
import re
44
from datetime import datetime, timedelta
55

6-
from mongoengine import *
7-
6+
from core.config.config import yeti_config
87
from core.config.mongoengine_extras import TimeDeltaField
98
from core.database import Node
109
from core.errors import TagValidationError
1110
from core.helpers import iterify
11+
from mongoengine import *
1212

1313

1414
class Tag(Node):
@@ -17,7 +17,9 @@ class Tag(Node):
1717
created = DateTimeField(default=datetime.utcnow)
1818
produces = ListField(ReferenceField("Tag", reverse_delete_rule=PULL))
1919
replaces = ListField(StringField())
20-
default_expiration = TimeDeltaField(default=timedelta(days=90))
20+
default_expiration = TimeDeltaField(
21+
default=timedelta(seconds=yeti_config.get("tag", "default_tag_expiration"))
22+
)
2123

2224
meta = {"ordering": ["name"], "indexes": ["name", "replaces"]}
2325

@@ -61,7 +63,9 @@ class ObservableTag(EmbeddedDocument):
6163
name = StringField(required=True)
6264
first_seen = DateTimeField(default=datetime.utcnow)
6365
last_seen = DateTimeField(default=datetime.utcnow)
64-
expiration = TimeDeltaField(default=timedelta(days=90))
66+
expiration = TimeDeltaField(
67+
default=timedelta(seconds=yeti_config.get("tag", "default_tag_expiration"))
68+
)
6569
fresh = BooleanField(default=True)
6670

6771
def __unicode__(self):

‎core/observables/url.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ class Url(Observable):
2020
regex = (
2121
r"(?P<search>((?P<scheme>[\w]{2,9}):\/\/)?([\S]*\:[\S]*\@)?(?P<hostname>"
2222
+ Hostname.main_regex
23-
+ ")(\:[\d]{1,5})?(?P<path>(\/[\S]*)?(\?[\S]*)?(\#[\S]*)?))"
23+
+ r")(\:[\d]{1,5})?(?P<path>(\/[^\?]*?)?(\?[^#]*?)?(\#.*?)?))"
2424
)
2525
search_regex = (
2626
r"(?P<search>((?P<scheme>[\w]{2,9}):\/\/)?([\S]*\:[\S]*\@)?(?P<hostname>"
2727
+ Hostname.main_regex
28-
+ ")(\:[\d]{1,5})?(?P<path>((\/[\S]*)?(\?[\S]*)?(\#[\S]*)?)[\w/])?)"
28+
+ r")(\:[\d]{1,5})?(?P<path>((\/[^\?]*?)?(\?[^#]*?)?(\#.*?)?)[\w/])?)"
2929
)
3030

3131
DISPLAY_FIELDS = Observable.DISPLAY_FIELDS + [

‎core/web/api/export.py

+4
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ def toggle(self, id):
8989

9090
def _parse_request(self, json):
9191
params = json
92+
if "fresh_tags" in params:
93+
params["fresh_tags"] = True
94+
else:
95+
params["fresh_tags"] = False
9296
params["frequency"] = string_to_timedelta(params.get("frequency", "1:00:00"))
9397
params["ignore_tags"] = [
9498
Tag.objects.get(name=name.strip())

‎core/web/api/templates/export_api.html

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
<td>{{ macros.display_tags(export["ignore_tags"]) }}</td>
1717
<td>{{ macros.display_tags(export["include_tags"]) }}</td>
1818
<td>{{ macros.display_tags(export["exclude_tags"]) }}</td>
19+
<td>{{ export["fresh_tags"] }}</td>
1920
<td>{{ export["template"]["name"] }}</td>
2021
<td class="status">{{ export["status"] or "N/A"}}</td>
2122
{% if toggle %}

‎core/web/frontend/observables.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,16 @@ def index(self):
8484
if request.method == "POST":
8585
lines = []
8686
obs = {}
87+
8788
if request.files.get("bulk-file"): # request files
88-
lines = request.files.get("bulk-file").readlines()
89+
lines = request.files.get("bulk-file").read()
8990
else:
90-
lines = request.form["bulk-text"].split("\n")
91+
lines = request.form["bulk-text"]
92+
93+
if type(lines) is bytes:
94+
lines = lines.decode()
95+
96+
lines = lines.split("\n")
9197

9298
invalid_observables = 0
9399
if bool(request.form.get("add", False)) and current_user.has_permission(

‎core/web/frontend/templates/dataflows.html

+3
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ <h5>New export</h5>
8888
{% endfor %}
8989
</select>
9090
</div>
91+
<div class="form-group">
92+
<input type="checkbox" name="fresh_tags" checked> <label for="fresh_tags">Only consider <code>fresh</code> tags in include and exclude</label>
93+
</div>
9194
<button type="button" id="yeti-new-button" data-url="{{ url_for('api.Export:index') }}" class="btn btn-default btn-sm yeti-new" name="save">New</button>
9295
<button type="button" id="yeti-save-button" data-url="{{ url_for('api.Export:index') }}" class="btn btn-primary btn-sm yeti-save" data-clear="true" name="save">Save</button>
9396
<button type="button" id="yeti-delete-button" data-url="{{ url_for('api.Export:index') }}" class="btn btn-danger btn-sm yeti-delete" name="delete" disabled="disabled">Delete</button>

‎core/web/webapp.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,9 @@
2929
auth_module = import_module("core.auth.%s" % yeti_config.auth.module)
3030
webapp.register_blueprint(auth_module.auth)
3131
is_true = False
32-
print(yeti_config.mongodb)
32+
3333
if yeti_config.mongodb.tls:
3434
is_true = True
35-
open("logs.txt", "w").write("ssl ok \n")
3635

3736
connect(
3837
yeti_config.mongodb.database,

0 commit comments

Comments
 (0)