Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit aee43d5

Browse files
authoredMar 9, 2023
Admin features extension (#120)
* Added started at and finished at on attribute and embedding table * Added started at and finished at in the queries * Removed print * Added started_at and finished_at as for import and export * Added full name to task * Started at and finished at added on attributes * Added finished at for embeddings * Added finished at for sample projects * Finised at date * Finished at fix * Adding embeddings one by one when importing project * Embeddings import fix * Sleep outside for loop * Sleep back in the loop with reduced time * Submodules merge
1 parent 1f5e5e7 commit aee43d5

File tree

6 files changed

+75
-14
lines changed

6 files changed

+75
-14
lines changed
 
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
"""Added started_at and finished_at on attribute and embedding table
2+
3+
Revision ID: 546e5cd7feaa
4+
Revises: 3b118e1e02cb
5+
Create Date: 2023-03-06 12:21:19.095488
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
12+
# revision identifiers, used by Alembic.
13+
revision = '546e5cd7feaa'
14+
down_revision = '3b118e1e02cb'
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
# ### commands auto generated by Alembic - please adjust! ###
21+
op.add_column('attribute', sa.Column('started_at', sa.DateTime(), nullable=True))
22+
op.add_column('attribute', sa.Column('finished_at', sa.DateTime(), nullable=True))
23+
op.add_column('embedding', sa.Column('started_at', sa.DateTime(), nullable=True))
24+
op.add_column('embedding', sa.Column('finished_at', sa.DateTime(), nullable=True))
25+
# ### end Alembic commands ###
26+
27+
28+
def downgrade():
29+
# ### commands auto generated by Alembic - please adjust! ###
30+
op.drop_column('embedding', 'finished_at')
31+
op.drop_column('embedding', 'started_at')
32+
op.drop_column('attribute', 'finished_at')
33+
op.drop_column('attribute', 'started_at')
34+
# ### end Alembic commands ###

‎controller/attribute/manager.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from util import daemon, notification
1212

1313
from . import util
14+
from sqlalchemy import sql
1415

1516

1617
def get_attribute(project_id: str, attribute_id: str) -> Attribute:
@@ -91,7 +92,6 @@ def update_attribute(
9192
source_code: str,
9293
visibility: str,
9394
) -> None:
94-
9595
attribute_item: Attribute = attribute.update(
9696
project_id,
9797
attribute_id,
@@ -142,7 +142,6 @@ def add_running_id(
142142
project_id, attribute_name, for_retokenization, with_commit=True
143143
)
144144
if for_retokenization:
145-
146145
daemon.run(
147146
request_tokenize_project,
148147
project_id,
@@ -192,6 +191,7 @@ def calculate_user_attribute_all_records(
192191
attribute_id=attribute_id,
193192
state=AttributeState.RUNNING.value,
194193
with_commit=True,
194+
started_at=sql.func.now(),
195195
)
196196
notification.send_organization_update(
197197
project_id=project_id, message=f"calculate_attribute:started:{attribute_id}"
@@ -207,7 +207,6 @@ def calculate_user_attribute_all_records(
207207
def __calculate_user_attribute_all_records(
208208
project_id: str, user_id: str, attribute_id: str
209209
) -> None:
210-
211210
try:
212211
calculated_attributes = util.run_attribute_calculation_exec_env(
213212
attribute_id=attribute_id, project_id=project_id, doc_bin="docbin_full"
@@ -285,6 +284,7 @@ def __calculate_user_attribute_all_records(
285284
attribute_id=attribute_id,
286285
state=AttributeState.USABLE.value,
287286
with_commit=True,
287+
finished_at=sql.func.now(),
288288
)
289289

290290
notification.send_organization_update(

‎controller/embedding/manager.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ def get_recommended_encoders() -> List[Any]:
1616
else:
1717
existing_models = []
1818
for model in existing_models:
19-
2019
if not model["zero_shot_pipeline"]:
2120
not_yet_known = (
2221
len(
@@ -110,6 +109,6 @@ def __embed_one_by_one_helper(
110109
user_id=user_id,
111110
config_string=splitted[2],
112111
)
113-
time.sleep(10)
112+
time.sleep(5)
114113
while util.has_encoder_running(project_id):
115-
time.sleep(10)
114+
time.sleep(5)

‎controller/transfer/project_transfer_manager.py

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from util.notification import create_notification
3434
from submodules.s3 import controller as s3
3535
import os
36+
from sqlalchemy import sql
3637

3738
logging.basicConfig(level=logging.INFO)
3839
logger = logging.getLogger(__name__)
@@ -149,20 +150,30 @@ def import_file(
149150
"""
150151
send_progress_update_throttle(project_id, task_id, 0)
151152
project_item = project.get(project_id)
152-
project_item.name = data.get("project_details_data",).get(
153+
project_item.name = data.get(
154+
"project_details_data",
155+
).get(
153156
"name",
154157
)
155-
project_item.description = data.get("project_details_data",).get(
158+
project_item.description = data.get(
159+
"project_details_data",
160+
).get(
156161
"description",
157162
)
158-
project_item.tokenizer = data.get("project_details_data",).get(
163+
project_item.tokenizer = data.get(
164+
"project_details_data",
165+
).get(
159166
"tokenizer",
160167
)
161-
spacy_language = data.get("project_details_data",).get(
168+
spacy_language = data.get(
169+
"project_details_data",
170+
).get(
162171
"tokenizer",
163172
)[:2]
164173
project_item.tokenizer_blank = spacy_language
165-
project_item.status = data.get("project_details_data",).get(
174+
project_item.status = data.get(
175+
"project_details_data",
176+
).get(
166177
"status",
167178
)
168179
old_project_id = data.get(
@@ -209,6 +220,8 @@ def import_file(
209220
"source_code",
210221
),
211222
visibility=attribute_item.get("visibility"),
223+
started_at=attribute_item.get("started_at"),
224+
finished_at=attribute_item.get("finished_at"),
212225
project_id=project_id,
213226
)
214227
attribute_ids_by_old_id[
@@ -606,7 +619,6 @@ def import_file(
606619
for embedding_item in data.get(
607620
"embeddings_data",
608621
):
609-
610622
attribute_id = embedding_item.get("attribute_id")
611623
embedding_name = embedding_item.get("name")
612624
if attribute_id:
@@ -617,6 +629,10 @@ def import_file(
617629
)
618630
attribute_id = attribute_ids_by_old_name[attribute_name]
619631

632+
finished_at_str = "finished_at" in embedding_item
633+
if not finished_at_str:
634+
embedding_item["finished_at"] = sql.func.now()
635+
620636
embedding_object = embedding.create(
621637
project_id=project_id,
622638
attribute_id=attribute_id,
@@ -628,6 +644,12 @@ def import_file(
628644
type=embedding_item.get(
629645
"type",
630646
),
647+
started_at=embedding_item.get(
648+
"started_at",
649+
),
650+
finished_at=embedding_item.get(
651+
"finished_at",
652+
),
631653
)
632654
embedding_ids[
633655
embedding_item.get(
@@ -987,6 +1009,8 @@ def get_project_export_dump(
9871009
"state": attribute_item.state,
9881010
"logs": attribute_item.logs,
9891011
"visibility": attribute_item.visibility,
1012+
"started_at": attribute_item.started_at,
1013+
"finished_at": attribute_item.finished_at,
9901014
}
9911015
for attribute_item in attributes
9921016
]
@@ -1080,6 +1104,8 @@ def get_project_export_dump(
10801104
"name": embedding_item.name,
10811105
"custom": embedding_item.custom,
10821106
"type": embedding_item.type,
1107+
"started_at": embedding_item.started_at,
1108+
"finished_at": embedding_item.finished_at,
10831109
}
10841110
for embedding_item in embeddings
10851111
]
@@ -1218,7 +1244,7 @@ def delete_project(project_id: str) -> bool:
12181244

12191245
def replace_by_mappings(text: str, mappings: List[Dict[str, str]]) -> str:
12201246
for mapping in mappings:
1221-
for (key, value) in mapping.items():
1247+
for key, value in mapping.items():
12221248
text = text.replace(str(key), str(value))
12231249
return text
12241250

‎graphql_api/types.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,3 +801,5 @@ class Task(graphene.ObjectType):
801801
project_name = graphene.String()
802802
state = graphene.String()
803803
task_type = graphene.String()
804+
started_at = graphene.DateTime()
805+
finished_at = graphene.DateTime()

0 commit comments

Comments
 (0)
Please sign in to comment.