Skip to content

Commit d71f215

Browse files
JarryShawJarryShaw
committed
New distribution [0.9.5]
 * revised hostname related models (FQDN max-length) * updated documentations & demos
1 parent 24d3d39 commit d71f215

File tree

11 files changed

+78
-15
lines changed

11 files changed

+78
-15
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ LABEL org.opencontainers.image.title="darc" \
44
org.opencontainers.image.description="Darkweb Crawler Project" \
55
org.opencontainers.image.url="https://darc.jarryshaw.me/" \
66
org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
7-
org.opencontainers.image.version="0.9.4.post2" \
7+
org.opencontainers.image.version="0.9.5" \
88
org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
99

1010
STOPSIGNAL SIGINT

‎darc/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@
4040
from darc.sites import register as register_sites # pylint: disable=unused-import
4141

4242
__all__ = ['darc']
43-
__version__ = '0.9.4.post2'
43+
__version__ = '0.9.5'

‎darc/model/tasks/hostname.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from typing import TYPE_CHECKING
1818

19-
from peewee import DateTimeField, TextField
19+
from peewee import CharField, DateTimeField
2020

2121
from darc.model.abc import BaseModel
2222

@@ -30,6 +30,6 @@ class HostnameQueueModel(BaseModel):
3030
"""Hostname task queue."""
3131

3232
#: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
33-
hostname: str = TextField()
33+
hostname: str = CharField(max_length=255, unique=True) # a valid FQDN is at most 255 characters
3434
#: Timestamp of last update.
3535
timestamp: 'datetime' = DateTimeField()

‎darc/model/web/hostname.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
from typing import TYPE_CHECKING
1616

17-
from peewee import DateTimeField, TextField
17+
from peewee import CharField, DateTimeField
1818

1919
from darc._compat import cached_property
2020
from darc._typing import SPHINX_BUILD
@@ -65,8 +65,10 @@ class HostnameModel(BaseModel):
6565
#: :attr:`URLModel.hostname <darc.model.web.url.URLModel.hostname>`.
6666
urls: 'List[URLModel]'
6767

68-
#: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
69-
hostname: str = TextField()
68+
#: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`). The maximum length of
69+
#: the host name and of the fully qualified domain name (FQDN) is 63 bytes per
70+
#: label and 255 characters per FQDN.
71+
hostname: str = CharField(max_length=255, unique=True) # a valid FQDN is at most 255 characters
7072
#: Proxy type (c.f. :attr:`link.proxy <darc.link.Link.proxy>`).
7173
proxy: Proxy = IntEnumField(choices=Proxy)
7274

‎debug.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.title="darc" \
1515
org.opencontainers.image.description="Darkweb Crawler Project" \
1616
org.opencontainers.image.url="https://darc.jarryshaw.me/" \
1717
org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
18-
org.opencontainers.image.version="0.9.4.post2" \
18+
org.opencontainers.image.version="0.9.5" \
1919
org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
2020
#EXPOSE 9050
2121

‎demo/model.py‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# -*- coding: utf-8 -*-
22
# pylint: disable=ungrouped-imports
33

4+
import enum
45
import os
56
from typing import TYPE_CHECKING
67

@@ -10,12 +11,67 @@
1011

1112
if TYPE_CHECKING:
1213
from datetime import datetime
13-
from typing import Any, Dict, List
14+
from enum import IntEnum
15+
from typing import Any, Dict, List, Optional
1416

1517
# database client
1618
DB = playhouse.db_url.connect(os.getenv('DB_URL', 'mysql://127.0.0.1'))
1719

1820

21+
class IntEnumField(peewee.IntegerField):
22+
""":class:`enum.IntEnum` data field."""
23+
24+
#: The original :class:`enum.IntEnum` class.
25+
choices: 'IntEnum'
26+
27+
# def db_value(self, value: 'Optional[IntEnum]') -> 'Optional[str]': # pylint: disable=inconsistent-return-statements
28+
# """Dump the value for database storage.
29+
30+
# Args:
31+
# val: Original enumeration object.
32+
33+
# Returns:
34+
# Integral representation of the enumeration.
35+
36+
# """
37+
# if value is not None:
38+
# return value
39+
40+
def python_value(self, value: 'Optional[int]') -> 'Optional[IntEnum]': # pylint: disable=inconsistent-return-statements
41+
"""Load the value from database storage.
42+
43+
Args:
44+
value: Integral representation of the enumeration.
45+
46+
Returns:
47+
Original enumeration object.
48+
49+
"""
50+
if value is not None:
51+
return self.choices(value) # type: ignore
52+
return None
53+
54+
55+
class Proxy(enum.IntEnum):
56+
"""Proxy types supported by :mod:`darc`.
57+
58+
.. _tor2web: https://onion.sh/
59+
"""
60+
61+
#: No proxy.
62+
NULL = enum.auto()
63+
#: Tor proxy.
64+
TOR = enum.auto()
65+
#: I2P proxy.
66+
I2P = enum.auto()
67+
#: ZeroNet proxy.
68+
ZERONET = enum.auto()
69+
#: Freenet proxy.
70+
FREENET = enum.auto()
71+
#: Proxied Tor (`tor2web`_, no proxy).
72+
TOR2WEB = enum.auto()
73+
74+
1975
def table_function(model_class: peewee.Model) -> str:
2076
"""Generate table name dynamically.
2177
@@ -81,9 +137,9 @@ class HostnameModel(BaseModel):
81137
"""Data model for a hostname record."""
82138

83139
#: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
84-
hostname: str = peewee.TextField()
140+
hostname: str = peewee.CharField(max_length=255, unique=True) # a valid FQDN is at most 255 characters
85141
#: Proxy type (c.f. :attr:`link.proxy <darc.link.Link.proxy>`).
86-
proxy: str = peewee.CharField(max_length=8)
142+
proxy: 'Proxy' = IntEnumField(choices=Proxy)
87143

88144
#: Timestamp of first ``new_host`` submission.
89145
discovery: 'datetime' = peewee.DateTimeField()

‎docker/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ LABEL org.opencontainers.image.title="darc" \
44
org.opencontainers.image.description="Darkweb Crawler Project" \
55
org.opencontainers.image.url="https://darc.jarryshaw.me/" \
66
org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
7-
org.opencontainers.image.version="0.9.4.post2" \
7+
org.opencontainers.image.version="0.9.5" \
88
org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
99

1010
STOPSIGNAL SIGINT

‎docker/debug.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.title="darc" \
1515
org.opencontainers.image.description="Darkweb Crawler Project" \
1616
org.opencontainers.image.url="https://darc.jarryshaw.me/" \
1717
org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
18-
org.opencontainers.image.version="0.9.4.post2" \
18+
org.opencontainers.image.version="0.9.5" \
1919
org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
2020
#EXPOSE 9050
2121

‎docs/source/conf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
author = 'Jarry Shaw'
3838

3939
# The full version, including alpha/beta/rc tags
40-
release = '0.9.4.post2'
40+
release = '0.9.5'
4141

4242

4343
# -- General configuration ---------------------------------------------------

‎docs/source/demo/model.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,10 @@ Assuming the database is using |peewee|_ as ORM and
1111
.. _peewee: https://docs.peewee-orm.com/
1212
.. _MySQL: https://mysql.com/
1313

14+
.. important::
15+
16+
For more updated, battlefield-tested version of the
17+
data models, please refer to :mod:`darc.model.web`.
18+
1419
.. literalinclude:: ../../../demo/model.py
1520
:language: python

0 commit comments

Comments
 (0)