Skip to content

Commit db9628b

Browse files
sandyhidersandyhider
authored
Merge pull request #111 from jhuapl-boss/integration
Integration
2 parents c2e26d2 + 3790608 commit db9628b

22 files changed

+519
-123
lines changed

‎.circleci/config.yml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
version: 2.1
2+
orbs:
3+
python: circleci/[email protected]
4+
slack: circleci/[email protected]
5+
6+
commands:
7+
# Should be able to remove this when 2.1 released on PyPi.
8+
install_heaviside:
9+
description: "Get heaviside from GitHub"
10+
steps:
11+
- run: git clone https://github.com/jhuapl-boss/heaviside.git
12+
- run: cd heaviside && pip install -r requirements.txt
13+
- run: cd heaviside && pip install .
14+
15+
install_ndingest:
16+
description: "Get ndingest from GitHub"
17+
steps:
18+
- run: git clone https://github.com/jhuapl-boss/ndingest.git
19+
- run: pip install -e ndingest/.
20+
- run: cp ndingest/ndingest/settings/settings.ini.test ndingest/ndingest/settings/settings.ini
21+
22+
install_spdb:
23+
description: "Clone spdb and install"
24+
steps:
25+
- run: git clone https://github.com/jhuapl-boss/spdb.git
26+
- run: cd spdb && python setup.py build_ext
27+
- run: cd spdb && pip install .
28+
- run: cd spdb && pip install -r requirements-test.txt
29+
30+
install_bossutils:
31+
description: "The bossutils folder of the project is pip installable"
32+
steps:
33+
- run: git clone https://github.com/jhuapl-boss/boss-tools.git
34+
- run: cd boss-tools && git checkout integration
35+
- run: cd boss-tools/bossutils && pip install .
36+
37+
install_compressed_segmentation:
38+
# Compile this locally so we know that it's built using the same version
39+
# of numpy used by the rest of the Boss endpoint. This is a dependendy
40+
# of cloud_volume.
41+
description: "Compile and install compressed_segmentation package"
42+
steps:
43+
- run: sudo apt install -y g++ python3-dev
44+
# Use --no-binary so it's built with our version of Numpy.
45+
- run: pip install --no-binary compressed_segmentation compressed_segmentation
46+
47+
install_cvdb:
48+
description: "Clone cvdb and install"
49+
steps:
50+
- run: git clone https://github.com/jhuapl-boss/cvdb.git
51+
########## Change this to integration when the branch is merged!! ######
52+
- run: cd cvdb && git checkout pin-cloudvolume
53+
- run: cd cvdb && pip install .
54+
55+
install_ingest_client:
56+
description: "Clone ingest-client and install"
57+
# ToDo: consider adding this to Boss' requirements.txt.
58+
steps:
59+
- run: git clone https://github.com/jhuapl-boss/ingest-client.git
60+
- run: cd ingest-client && pip install .
61+
62+
install:
63+
description: "Install Python dependencies"
64+
steps:
65+
- checkout
66+
- python/load-cache
67+
# Consider moving all these to boss' requirements.txt!
68+
- install_heaviside
69+
- install_ndingest
70+
- install_spdb
71+
- install_bossutils
72+
- install_compressed_segmentation
73+
- install_cvdb
74+
- install_ingest_client
75+
# Install boss' dependencies last because Blosc isn't pinned in intern.
76+
- python/install-deps
77+
- python/save-cache
78+
79+
setup:
80+
description: "Miscellaneous setup for testing"
81+
steps:
82+
- run: sudo mkdir -p /var/log/boss
83+
- run: sudo chown circleci /var/log/boss
84+
- run: sudo mkdir -p /etc/boss
85+
- run: sudo chown circleci /etc/boss
86+
- run: cp /home/circleci/project/django/boss-test.config /etc/boss/boss.config
87+
88+
wait_for_mysql:
89+
description: "Ensure MySQL container ready"
90+
steps:
91+
- run:
92+
name: Wait
93+
command: |
94+
for i in `seq 1 10`;
95+
do
96+
nc -z 127.0.0.1 3306 && echo Success && exit 0
97+
echo -n .
98+
sleep 1
99+
done
100+
echo Failed waiting for MySQL && exit 1
101+
102+
do_django_migrations:
103+
description: "Make any necessary DB migrations"
104+
steps:
105+
- run: cd django && python manage.py makemigrations --settings boss.settings.mysql
106+
- run: cd django && python manage.py migrate --settings boss.settings.mysql
107+
108+
test_django:
109+
description: "Run Boss unit tests"
110+
steps:
111+
# Don't use the Django-nose2 test runner because it loads the layers used
112+
# for integration tests.
113+
- run: cd django && python manage.py test --settings boss.settings.mysql --testrunner=django.test.runner.DiscoverRunner
114+
115+
jobs:
116+
test_py3_9:
117+
docker:
118+
- image: circleci/python:3.9
119+
120+
- image: circleci/mysql:5.7.37-ram
121+
environment:
122+
MYSQL_ROOT_PASSWORD: MICrONS
123+
MYSQL_DATABASE: microns
124+
125+
steps:
126+
- install
127+
- setup
128+
- wait_for_mysql
129+
- do_django_migrations
130+
- test_django
131+
- slack/notify:
132+
event: fail
133+
template: basic_fail_1
134+
- slack/notify:
135+
event: pass
136+
template: basic_success_1
137+
138+
workflows:
139+
test:
140+
jobs:
141+
- test_py3_9:
142+
context:
143+
- circleci

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Boss Changelog
2+
3+
## Unreleased
4+
* Improvements
5+
- Volumetric ingests complete and clean up properly from the ingest client.
6+
7+
## 1.0.7
8+
* Improvements
9+
- Hiding Resource Delete buttons in mgmt console if the user does not have permissions to delete.

‎django/boss-test.config‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# For CI testing on CircleCI, this file copied into /etc/boss on the Docker
2+
# container.
3+
4+
[system]
5+
fqdn = endpoint.test.boss
6+
type = endpoint
7+
8+
[consul]
9+
cluster =
10+
11+
[vault]
12+
url = http://vault:8200
13+
token =
14+
15+
[aws_mngr]
16+
num_sessions = auto
17+
18+
[aws]
19+
db = endpoint-db.test.boss
20+
meta-db = bossmeta.test.boss
21+
cache = cache.test.boss
22+
cache-state = cache-state.test.boss
23+
cache-db = 0
24+
cache-state-db = 0
25+
s3-bucket =
26+
s3-index-table = s3index.test.boss
27+
s3-flush-queue = https://sqs.us-east-1.amazonaws.com/8395829333/S3flushtestBoss
28+
s3-flush-deadletter-queue = https://sqs.us-east-1.amazonaws.com/8395829333/DeadlettertestBoss
29+
cuboid_bucket = cuboids.test.boss
30+
tile_bucket = tiles.test.boss
31+
ingest_bucket = ingest.test.boss
32+
tile-index-table = tileindex.test.boss
33+
id-index-table = idIndex.test.boss
34+
id-count-table = idCount.test.boss
35+
sns-write-locked = some_sqs_arn
36+
downsample-queue = https://sqs.us-east-1.amazonaws.com/8395829333/downsampleQueueBoss
37+
38+
[auth]
39+
OIDC_VERIFY_SSL = True
40+
41+
[lambda]
42+
page_in_function = multiLambda-test-boss
43+
flush_function = multiLambda-test-boss
44+
ingest_function = multiLambda-test-boss
45+
downsample_volume = downsample-lambda
46+
tile_uploaded_function = foo
47+
48+
[sfn]
49+
downsample_volume_sfn = downsample_vol
50+
downsample_sfn = downsample

‎django/boss/urls.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@
5353
#url(r'^docs/', include('rest_framework_swagger.urls')),
5454
url(r'^docs/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
5555
url(r'^ping/', views.Ping.as_view()),
56-
url(r'^token/', views.Token.as_view()),
5756
url(r'^metric/', views.Metric.as_view()),
5857

5958
# deprecated urls

‎django/boss/views.py‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -102,44 +102,7 @@ def put(self, request):
102102
return BossHTTPError(" This API version is unsupported. Update to version {}".format(version),
103103
ErrorCodes.UNSUPPORTED_VERSION)
104104

105-
from django.http import HttpResponse, HttpResponseRedirect
106-
from django.views.generic import View
107-
108-
# import as to deconflict with our Token class
109-
from rest_framework.authtoken.models import Token as TokenModel
110-
111-
class Token(LoginRequiredMixin, View):
112-
def get(self, request):
113-
action = request.GET.get('action', None)
114-
115-
try:
116-
token = TokenModel.objects.get(user = request.user)
117-
if action == "Revoke":
118-
token.delete()
119-
token = None
120-
except:
121-
if action == "Generate":
122-
token = TokenModel.objects.create(user = request.user)
123-
else:
124-
token = None
125-
126-
if token is None:
127-
content = ""
128-
button = "Generate"
129-
else:
130-
content = "<textarea>{}</textarea>".format(token)
131-
button = "Revoke"
132-
133-
html = """
134-
<html>
135-
<head><title>BOSS Token Management</title></head>
136-
<body>
137-
{1}
138-
<a href="{0}?action={2}">{2}</a>
139-
</body>
140-
</html>
141-
""".format(request.path_info, content, button)
142-
return HttpResponse(html)
105+
from django.http import HttpResponse
143106

144107
from boss.throttling import MetricDatabase
145108
from bosscore.constants import ADMIN_USER

‎django/bosscore/test/setup_db.py‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,19 @@
3838
EXP22 = 'exp22'
3939
EXP_BASE_RES = 'exp-base-res-test'
4040
TEST_DATA_EXPERIMENTS = [EXP1, EXP22, EXP_BASE_RES]
41+
EXP_NOT_PUBLIC = 'exp-not-public'
4142

4243
CHAN_BASE_RES = 'chan-with-base-res'
44+
CHAN_NOT_PUBLIC = 'chan-not-public'
45+
46+
# Channel for cloudvolume tests uses this bucket name.
47+
CLOUD_VOL_BUCKET = 'bossdb-test-data'
48+
CVPATH_CHAN1 = 'col1/exp1/chan1'
49+
CVPATH_CHAN2 = 'col1/exp1/chan2'
50+
CVPATH_ANNO1 = 'col1/exp1/anno1'
51+
52+
# Collection names.
53+
COLL_NOT_PUBLIC = 'col-not-public'
4354

4455
class SetupTestDB:
4556
def __init__(self, super_user=None):
@@ -127,18 +138,21 @@ def insert_test_data(self):
127138
self.add_collection('col1', 'Description for collection1')
128139
self.add_collection('col1-22', 'Description for collection1-22')
129140
self.add_collection('col2', 'Description for collection2')
141+
self.add_collection(COLL_NOT_PUBLIC, 'Collection to test setting public', public=False)
130142

131143
self.add_coordinate_frame('cf1', 'Description for cf1', 0, 1000, 0, 1000, 0, 1000, 4, 4, 4)
132144

133145
self.add_experiment('col1', EXP1, 'cf1', NUM_HIERARCHY_LEVELS, 10, 1)
134146
self.add_experiment('col1', EXP22, 'cf1', NUM_HIERARCHY_LEVELS, 500, 1)
135147
self.add_experiment('col1', EXP_BASE_RES, 'cf1', NUM_HIERARCHY_LEVELS, 10, 1)
148+
self.add_experiment(COLL_NOT_PUBLIC, EXP_NOT_PUBLIC, 'cf1', NUM_HIERARCHY_LEVELS, 1, 1, public=False)
136149

137150
self.add_channel('col1', EXP1, 'channel1', 0, 0, 'uint8', 'image')
138151
self.add_channel('col1', EXP1, 'channel2', 0, 0, 'uint8', 'image')
139152
self.add_channel('col1', EXP1, 'channel3', 0, 0, 'uint64', 'annotation', ['channel1'])
140153
self.add_channel('col1', EXP_BASE_RES, CHAN_BASE_RES, 0, BASE_RESOLUTION, 'uint8', 'image')
141154
self.add_channel('col1', EXP1, 'layer1', 0, 0, 'uint64', 'annotation', ['channel1'])
155+
self.add_channel(COLL_NOT_PUBLIC, EXP_NOT_PUBLIC, CHAN_NOT_PUBLIC, 0, 0, 'uint8', 'image', public=False)
142156

143157
def insert_lookup_test_data(self):
144158
"""
@@ -149,7 +163,7 @@ def insert_lookup_test_data(self):
149163
self.add_collection('col2', 'Description for collection2')
150164

151165
self.add_coordinate_frame('cf1', 'Description for cf1', 0, 1000, 0, 1000, 0, 1000, 4, 4, 4)
152-
166+
153167
self.add_experiment('col1', 'exp1', 'cf1', 10, 10, 1)
154168

155169
# This experiment is _purposed_ named the same as the exp in col1.
@@ -183,22 +197,22 @@ def insert_cloudvolume_test_data(self):
183197
self.add_experiment('col1', 'exp1', 'cf1', 10, 500, 1)
184198

185199
# Dev Note: Prepopulated cloudvolume layer for uint8 data located at this cloudpath
186-
self.add_channel('col1', 'exp1', 'chan1', 0, 0, 'uint8', 'image',
187-
storage_type='cloudvol',
188-
bucket='bossdb-test-data',
189-
cv_path='col1/exp1/chan1')
190-
200+
self.add_channel('col1', 'exp1', 'chan1', 0, 0, 'uint8', 'image',
201+
storage_type='cloudvol',
202+
bucket=CLOUD_VOL_BUCKET,
203+
cv_path=CVPATH_CHAN1)
204+
191205
# Dev Note: Prepopulated cloudvolume layer for uint16 data located at this cloudpath
192-
self.add_channel('col1', 'exp1', 'chan2', 0, 0, 'uint16', 'image',
193-
storage_type='cloudvol',
194-
bucket='bossdb-test-data',
195-
cv_path='col1/exp1/chan2')
196-
206+
self.add_channel('col1', 'exp1', 'chan2', 0, 0, 'uint16', 'image',
207+
storage_type='cloudvol',
208+
bucket=CLOUD_VOL_BUCKET,
209+
cv_path=CVPATH_CHAN2)
210+
197211
# Dev Note: Prepopulated cloudvolume layer for uint16 data located at this cloudpath
198-
self.add_channel('col1', 'exp1', 'anno1', 0, 0, 'uint64', 'annotation',
199-
storage_type='cloudvol',
200-
bucket='bossdb-test-data',
201-
cv_path='col1/exp1/anno1')
212+
self.add_channel('col1', 'exp1', 'anno1', 0, 0, 'uint64', 'annotation',
213+
storage_type='cloudvol',
214+
bucket=CLOUD_VOL_BUCKET,
215+
cv_path=CVPATH_ANNO1)
202216

203217
def insert_ingest_test_data(self):
204218

0 commit comments

Comments
 (0)