Skip to content

Commit 8493a99

Browse files
yeraydiazdiazcooperlees
yeraydiazdiaz
authored and
cooperlees
committed
Pre-release filter (#83)
* Pre-release filter Grouped and renamed filter tests * Fix uninitialized `patterns` * Fix linting * Added `RegexProjectFilter` * Use `[filter_*]` config convention * Add `PreReleaseFilter` * Additional test and documentation * Use "bandersnatch" logger * Add test for blacklist release Use proper release naming
1 parent 151674b commit 8493a99

File tree

11 files changed

+418
-8
lines changed

11 files changed

+418
-8
lines changed

‎docs/filtering_configuration.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,41 @@ packages =
4646
example1
4747
example2>=1.4.2,<1.9,!=1.5.*,!=1.6.*
4848
```
49+
50+
### Prerelease filtering
51+
52+
Bandersnatch includes a plugin to filter our pre-releases of packages. To enable this plugin simply add `prerelease_release` to the enabled plugins list.
53+
54+
``` ini
55+
[blacklist]
56+
plugins =
57+
prerelease_release
58+
```
59+
60+
### Regex filtering
61+
62+
Advanced users who would like finer control over which packages and releases to filter can use the regex Bandersnatch plugin.
63+
64+
This plugin allows arbitrary regular expressions to be defined in the configuration, any package name or release version that matches will *not* be downloaded.
65+
66+
The plugin can be activated for packages and releases separately. For example to activate the project regex filter simply add it to the configuration as before:
67+
68+
``` ini
69+
[blacklist]
70+
plugins =
71+
regex_project
72+
```
73+
74+
If you'd like to filter releases using the regex filter use `regex_release` instead.
75+
76+
The regex plugin requires an extra section in the config to define the actual patterns to used for filtering:
77+
78+
``` ini
79+
[filter_regex]
80+
packages =
81+
.+-evil$
82+
releases =
83+
.+alpha\d$
84+
```
85+
86+
Note the same `filter_regex` section may include a `packages` and a `releases` entry with any number of regular expressions.

‎setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,11 @@
3030
[bandersnatch_filter_plugins.project]
3131
blacklist_project = bandersnatch_filter_plugins.blacklist_name:BlacklistProject
3232
whitelist_project = bandersnatch_filter_plugins.whitelist_name:WhitelistProject
33+
regex_project = bandersnatch_filter_plugins.regex_name:RegexProjectFilter
3334
[bandersnatch_filter_plugins.release]
3435
blacklist_release = bandersnatch_filter_plugins.blacklist_name:BlacklistRelease
36+
regex_release = bandersnatch_filter_plugins.regex_name:RegexReleaseFilter
37+
prerelease_release = bandersnatch_filter_plugins.prerelease_name:PreReleaseFilter
3538
[console_scripts]
3639
bandersnatch = bandersnatch.main:main
3740
[zc.buildout]

‎src/bandersnatch/filter.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ def filter_project_plugins() -> Iterable[Filter]:
110110
Returns
111111
-------
112112
list of bandersnatch.filter.Filter:
113-
List of objects drived from the bandersnatch.filter.Filter class
113+
List of objects derived from the bandersnatch.filter.Filter class
114114
"""
115115
return load_filter_plugins("bandersnatch_filter_plugins.project")
116116

@@ -122,6 +122,6 @@ def filter_release_plugins() -> Iterable[Filter]:
122122
Returns
123123
-------
124124
list of bandersnatch.filter.Filter:
125-
List of objects drived from the bandersnatch.filter.Filter class
125+
List of objects derived from the bandersnatch.filter.Filter class
126126
"""
127127
return load_filter_plugins("bandersnatch_filter_plugins.release")

‎src/bandersnatch/package.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -147,15 +147,15 @@ def sync(self, stop_on_error=False, attempts=3):
147147

148148
def _filter_releases(self):
149149
"""
150-
Run the release filtering plugins and remove any packages from the
151-
packages_to_sync that match any filters.
150+
Run the release filtering plugins and remove any releases from
151+
`releases` that match any filters.
152152
"""
153153
versions = list(self.releases.keys())
154154
for version in versions:
155-
filter = False
155+
filter_ = False
156156
for plugin in filter_release_plugins():
157-
filter = filter or plugin.check_match(name=self.name, version=version)
158-
if filter:
157+
filter_ = filter_ or plugin.check_match(name=self.name, version=version)
158+
if filter_:
159159
del (self.releases[version])
160160

161161
# TODO: async def once we go full asyncio - Have concurrency at the

src/bandersnatch/tests/test__bandersnatch_plugins__blacklist_name.py renamed to src/bandersnatch/tests/plugins/test_blacklist_name.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from bandersnatch.configuration import BandersnatchConfig
88
from bandersnatch.master import Master
99
from bandersnatch.mirror import Mirror
10+
from bandersnatch.package import Package
1011

1112

1213
class TestBlacklistProject(TestCase):
@@ -191,3 +192,26 @@ def test__plugin__loads__default(self):
191192
plugins = bandersnatch.filter.filter_release_plugins()
192193
names = [plugin.name for plugin in plugins]
193194
self.assertIn("blacklist_release", names)
195+
196+
def test__filter__matches__release(self):
197+
with open("test.conf", "w") as testconfig_handle:
198+
testconfig_handle.write(
199+
"""\
200+
[blacklist]
201+
plugins =
202+
blacklist_release
203+
packages =
204+
foo==1.2.0
205+
"""
206+
)
207+
instance = BandersnatchConfig()
208+
instance.config_file = "test.conf"
209+
instance.load_configuration()
210+
211+
mirror = Mirror(".", Master(url="https://foo.bar.com"))
212+
pkg = Package("foo", 1, mirror)
213+
pkg.releases = {"1.2.0": {}, "1.2.1": {}}
214+
215+
pkg._filter_releases()
216+
217+
self.assertEqual(pkg.releases, {"1.2.1": {}})

‎src/bandersnatch/tests/plugins/test_prerelease_name.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import os
2+
import re
3+
from collections import defaultdict
4+
from tempfile import TemporaryDirectory
5+
from unittest import TestCase
6+
7+
import bandersnatch.filter
8+
from bandersnatch.configuration import BandersnatchConfig
9+
from bandersnatch.master import Master
10+
from bandersnatch.mirror import Mirror
11+
from bandersnatch.package import Package
12+
from bandersnatch_filter_plugins import prerelease_name
13+
14+
15+
def _mock_config(contents, filename="test.conf"):
16+
"""
17+
Creates a config file with contents and loads them into a
18+
BandersnatchConfig instance.
19+
"""
20+
with open(filename, "w") as fd:
21+
fd.write(contents)
22+
23+
instance = BandersnatchConfig()
24+
instance.config_file = filename
25+
instance.load_configuration()
26+
return instance
27+
28+
29+
class BasePluginTestCase(TestCase):
30+
31+
tempdir = None
32+
cwd = None
33+
34+
def setUp(self):
35+
self.cwd = os.getcwd()
36+
self.tempdir = TemporaryDirectory()
37+
bandersnatch.filter.loaded_filter_plugins = defaultdict(list)
38+
os.chdir(self.tempdir.name)
39+
40+
def tearDown(self):
41+
if self.tempdir:
42+
os.chdir(self.cwd)
43+
self.tempdir.cleanup()
44+
self.tempdir = None
45+
46+
47+
class TestRegexReleaseFilter(BasePluginTestCase):
48+
49+
config_contents = """\
50+
[blacklist]
51+
plugins =
52+
prerelease_release
53+
"""
54+
55+
def test_plugin_includes_predefined_patterns(self):
56+
_mock_config(self.config_contents)
57+
58+
plugins = bandersnatch.filter.filter_release_plugins()
59+
60+
assert any(
61+
type(plugin) == prerelease_name.PreReleaseFilter for plugin in plugins
62+
)
63+
plugin = next(
64+
plugin
65+
for plugin in plugins
66+
if type(plugin) == prerelease_name.PreReleaseFilter
67+
)
68+
expected_patterns = [
69+
re.compile(pattern_string) for pattern_string in plugin.PRERELEASE_PATTERNS
70+
]
71+
assert plugin.patterns == expected_patterns
72+
73+
def test_plugin_check_match(self):
74+
_mock_config(self.config_contents)
75+
76+
bandersnatch.filter.filter_release_plugins()
77+
78+
mirror = Mirror(".", Master(url="https://foo.bar.com"))
79+
pkg = Package("foo", 1, mirror)
80+
pkg.releases = {
81+
"1.2.0alpha1": {},
82+
"1.2.0a2": {},
83+
"1.2.0beta1": {},
84+
"1.2.0b2": {},
85+
"1.2.0rc1": {},
86+
"1.2.0": {},
87+
}
88+
89+
pkg._filter_releases()
90+
91+
assert pkg.releases == {"1.2.0": {}}

‎src/bandersnatch/tests/plugins/test_regex_name.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import os
2+
import re
3+
from collections import defaultdict
4+
from tempfile import TemporaryDirectory
5+
from unittest import TestCase
6+
7+
import bandersnatch.filter
8+
from bandersnatch.configuration import BandersnatchConfig
9+
from bandersnatch.master import Master
10+
from bandersnatch.mirror import Mirror
11+
from bandersnatch.package import Package
12+
from bandersnatch_filter_plugins import regex_name
13+
14+
15+
def _mock_config(contents, filename="test.conf"):
16+
"""
17+
Creates a config file with contents and loads them into a
18+
BandersnatchConfig instance.
19+
"""
20+
with open(filename, "w") as fd:
21+
fd.write(contents)
22+
23+
instance = BandersnatchConfig()
24+
instance.config_file = filename
25+
instance.load_configuration()
26+
return instance
27+
28+
29+
class BasePluginTestCase(TestCase):
30+
31+
tempdir = None
32+
cwd = None
33+
34+
def setUp(self):
35+
self.cwd = os.getcwd()
36+
self.tempdir = TemporaryDirectory()
37+
bandersnatch.filter.loaded_filter_plugins = defaultdict(list)
38+
os.chdir(self.tempdir.name)
39+
40+
def tearDown(self):
41+
if self.tempdir:
42+
os.chdir(self.cwd)
43+
self.tempdir.cleanup()
44+
self.tempdir = None
45+
46+
47+
class TestRegexReleaseFilter(BasePluginTestCase):
48+
49+
config_contents = """\
50+
[blacklist]
51+
plugins =
52+
regex_release
53+
54+
[filter_regex]
55+
releases =
56+
.+rc\\d$
57+
.+alpha\\d$
58+
"""
59+
60+
def test_plugin_compiles_patterns(self):
61+
_mock_config(self.config_contents)
62+
63+
plugins = bandersnatch.filter.filter_release_plugins()
64+
65+
assert any(type(plugin) == regex_name.RegexReleaseFilter for plugin in plugins)
66+
plugin = next(
67+
plugin
68+
for plugin in plugins
69+
if type(plugin) == regex_name.RegexReleaseFilter
70+
)
71+
assert plugin.patterns == [re.compile(r".+rc\d$"), re.compile(r".+alpha\d$")]
72+
73+
def test_plugin_check_match(self):
74+
_mock_config(self.config_contents)
75+
76+
bandersnatch.filter.filter_release_plugins()
77+
78+
mirror = Mirror(".", Master(url="https://foo.bar.com"))
79+
pkg = Package("foo", 1, mirror)
80+
pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}
81+
82+
pkg._filter_releases()
83+
84+
assert pkg.releases == {"foo-1.2.0": {}}
85+
86+
87+
class TestRegexProjectFilter(BasePluginTestCase):
88+
89+
config_contents = """\
90+
[blacklist]
91+
plugins =
92+
regex_project
93+
94+
[filter_regex]
95+
packages =
96+
.+-evil$
97+
.+-neutral$
98+
"""
99+
100+
def test_plugin_compiles_patterns(self):
101+
_mock_config(self.config_contents)
102+
103+
plugins = bandersnatch.filter.filter_project_plugins()
104+
105+
assert any(type(plugin) == regex_name.RegexProjectFilter for plugin in plugins)
106+
plugin = next(
107+
plugin
108+
for plugin in plugins
109+
if type(plugin) == regex_name.RegexProjectFilter
110+
)
111+
assert plugin.patterns == [re.compile(r".+-evil$"), re.compile(r".+-neutral$")]
112+
113+
def test_plugin_check_match(self):
114+
_mock_config(self.config_contents)
115+
116+
bandersnatch.filter.filter_release_plugins()
117+
118+
mirror = Mirror(".", Master(url="https://foo.bar.com"))
119+
mirror.packages_to_sync = {"foo-good": {}, "foo-evil": {}, "foo-neutral": {}}
120+
mirror._filter_packages()
121+
122+
assert list(mirror.packages_to_sync.keys()) == ["foo-good"]

src/bandersnatch/tests/test__bandersnatch_plugins__whitelist_name.py renamed to src/bandersnatch/tests/plugins/test_whitelist_name.py

File renamed without changes.

‎src/bandersnatch_filter_plugins/blacklist_name.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def _determine_filtered_package_requirements(self):
125125

126126
def check_match(self, **kwargs):
127127
"""
128-
Check if the package name and version matches against a blacklisted
128+
Check if the package name and version matches against a blacklisted
129129
package version specifier.
130130
131131
Parameters

‎src/bandersnatch_filter_plugins/prerelease_name.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import logging
2+
import re
3+
4+
from bandersnatch.filter import FilterReleasePlugin
5+
6+
logger = logging.getLogger("bandersnatch")
7+
8+
9+
class PreReleaseFilter(FilterReleasePlugin):
10+
"""
11+
Filters releases considered pre-releases.
12+
"""
13+
14+
name = "prerelease_release"
15+
PRERELEASE_PATTERNS = (r".+rc\d$", r".+a(lpha)?\d$", r".+b(eta)?\d$")
16+
17+
def initialize_plugin(self):
18+
"""
19+
Initialize the plugin reading patterns from the config.
20+
"""
21+
self.patterns = [
22+
re.compile(pattern_string) for pattern_string in self.PRERELEASE_PATTERNS
23+
]
24+
25+
logger.info(f"Initialized prerelease plugin with {self.patterns}")
26+
27+
def check_match(self, name, version):
28+
"""
29+
Check if a release version matches any of the specificed patterns.
30+
31+
Parameters
32+
==========
33+
name: str
34+
Release name
35+
version: str
36+
Release version
37+
38+
Returns
39+
=======
40+
bool:
41+
True if it matches, False otherwise.
42+
"""
43+
return any(pattern.match(version) for pattern in self.patterns)

0 commit comments

Comments
 (0)