Skip to content

Commit 81d6425

Browse files
shreyanidshreyanid
authored
feat: add source permission metadata for google drive (#415)
* confluence doc permission * optional metadata * permissions client * clean gdrive, remove confluence * clean * refactor normalization structure * ignore domain and anyone * test fixture * version and types * dup permissions_data type update; overwrite fixtures * update all fixtures * merge fix * add sort for determinism; fixture update * empty case should be dict
1 parent cb1277a commit 81d6425

File tree

8 files changed

+2792
-6364
lines changed

8 files changed

+2792
-6364
lines changed

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
## 1.0.4
2+
3+
### Features
4+
5+
* **Normalize user and group permissions in Google Drive source connector**
6+
17
## 1.0.3
28

39
### Features

‎test/integration/connectors/expected_results/google_drive_source/file_data/1r-RDeDtKprFQWST4PCIPV618y_sBL7N7EEWg7q4kZrU.json

+23-32
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,28 @@
1515
"date_created": "1738365200.695",
1616
"date_modified": "1738601393.049",
1717
"date_processed": null,
18-
"permissions_data": [
19-
{
20-
"id": "03252897642318370246k",
21-
"displayName": "Unstructured AI",
22-
"type": "domain",
23-
"kind": "drive#permission",
24-
"role": "writer",
25-
"allowFileDiscovery": false,
26-
"domain": "unstructured.io"
18+
"permissions_data": {
19+
"read": {
20+
"users": [
21+
"03909661036819820693",
22+
"18298851591250030956"
23+
],
24+
"groups": []
2725
},
28-
{
29-
"id": "18298851591250030956",
30-
"displayName": "[email protected]",
31-
"type": "user",
32-
"kind": "drive#permission",
33-
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
34-
"emailAddress": "[email protected]",
35-
"role": "writer",
36-
"deleted": false,
37-
"pendingOwner": false
26+
"update": {
27+
"users": [
28+
"03909661036819820693",
29+
"18298851591250030956"
30+
],
31+
"groups": []
3832
},
39-
{
40-
"id": "03909661036819820693",
41-
"displayName": "bryan.chen",
42-
"type": "user",
43-
"kind": "drive#permission",
44-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjW_y8aRy0z-MgGdTg9ybw86182DzfQZCuS0P4b-1JkQNnHvzQ=s64",
45-
"emailAddress": "[email protected]",
46-
"role": "owner",
47-
"deleted": false,
48-
"pendingOwner": false
33+
"delete": {
34+
"users": [
35+
"03909661036819820693"
36+
],
37+
"groups": []
4938
}
50-
],
39+
},
5140
"filesize_bytes": null
5241
},
5342
"additional_metadata": {
@@ -84,7 +73,9 @@
8473
"canModifyLabels": false,
8574
"canModifyEditorContentRestriction": true,
8675
"canModifyOwnerContentRestriction": false,
87-
"canRemoveContentRestriction": false
76+
"canRemoveContentRestriction": false,
77+
"canDisableInheritedPermissions": false,
78+
"canEnableInheritedPermissions": true
8879
},
8980
"permissionIds": [
9081
"03252897642318370246k",
@@ -93,6 +84,6 @@
9384
]
9485
},
9586
"reprocess": false,
96-
"local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmp9lg5f7kk/ingest_integration_test_do_not_change",
87+
"local_download_path": "/private/var/folders/xb/08zztz_j57vgp3pqjx1g4jh40000gn/T/tmpsb63xpdz/ingest_integration_test_do_not_change",
9788
"display_name": null
9889
}

‎test_e2e/expected-structured-output/google-drive/fake.docx.json

+29-70
Original file line numberDiff line numberDiff line change
@@ -16,79 +16,38 @@
1616
},
1717
"date_created": "1686809759.687",
1818
"date_modified": "1686809743.0",
19-
"permissions_data": [
20-
{
21-
"id": "anyoneWithLink",
22-
"type": "anyone",
23-
"kind": "drive#permission",
24-
"role": "reader",
25-
"allowFileDiscovery": false
19+
"permissions_data": {
20+
"read": {
21+
"users": [
22+
"03887347926440898356",
23+
"04774006893477068632",
24+
"09147371668407854156",
25+
"13662041828528429192",
26+
"18298851591250030956"
27+
],
28+
"groups": [
29+
"10619079449796831495"
30+
]
2631
},
27-
{
28-
"id": "10619079449796831495",
29-
"displayName": "fuse-team",
30-
"type": "group",
31-
"kind": "drive#permission",
32-
"emailAddress": "[email protected]",
33-
"role": "writer",
34-
"deleted": false
32+
"update": {
33+
"users": [
34+
"03887347926440898356",
35+
"04774006893477068632",
36+
"09147371668407854156",
37+
"13662041828528429192",
38+
"18298851591250030956"
39+
],
40+
"groups": [
41+
"10619079449796831495"
42+
]
3543
},
36-
{
37-
"id": "03887347926440898356",
38-
"displayName": "michal.martyniak",
39-
"type": "user",
40-
"kind": "drive#permission",
41-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWWmJCSvduWur55hl36IxwKs5FJ1FWMoK6KtFlNUHDBU-McvlI=s64",
42-
"emailAddress": "[email protected]",
43-
"role": "writer",
44-
"deleted": false,
45-
"pendingOwner": false
46-
},
47-
{
48-
"id": "13662041828528429192",
49-
"displayName": "rob",
50-
"type": "user",
51-
"kind": "drive#permission",
52-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjV31Wb5kVmEk66cog1KN-N_twpHHoDttcCQ9pRvE1cz1-FLHQ=s64",
53-
"emailAddress": "[email protected]",
54-
"role": "writer",
55-
"deleted": false,
56-
"pendingOwner": false
57-
},
58-
{
59-
"id": "18298851591250030956",
60-
"displayName": "[email protected]",
61-
"type": "user",
62-
"kind": "drive#permission",
63-
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
64-
"emailAddress": "[email protected]",
65-
"role": "writer",
66-
"deleted": false,
67-
"pendingOwner": false
68-
},
69-
{
70-
"id": "04774006893477068632",
71-
"displayName": "ryan",
72-
"type": "user",
73-
"kind": "drive#permission",
74-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjXeWpu7QcZuYqIl3p1mwqzS8XGFJ4RqA3Xjljfkm1DcFZ9M7A=s64",
75-
"emailAddress": "[email protected]",
76-
"role": "owner",
77-
"deleted": false,
78-
"pendingOwner": false
79-
},
80-
{
81-
"id": "09147371668407854156",
82-
"displayName": "roman",
83-
"type": "user",
84-
"kind": "drive#permission",
85-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWoGrFCgXcF6CtiBIBLnAfM68qUnQaJOcgvg3qzfQ3W8Ch6dA=s64",
86-
"emailAddress": "[email protected]",
87-
"role": "writer",
88-
"deleted": false,
89-
"pendingOwner": false
44+
"delete": {
45+
"users": [
46+
"04774006893477068632"
47+
],
48+
"groups": []
9049
}
91-
],
50+
},
9251
"filesize_bytes": 36602
9352
}
9453
}

‎test_e2e/expected-structured-output/google-drive/nested/fake.docx.json

+29-70
Original file line numberDiff line numberDiff line change
@@ -16,79 +16,38 @@
1616
},
1717
"date_created": "1718722775.76",
1818
"date_modified": "1718722788.018",
19-
"permissions_data": [
20-
{
21-
"id": "anyoneWithLink",
22-
"type": "anyone",
23-
"kind": "drive#permission",
24-
"role": "reader",
25-
"allowFileDiscovery": false
19+
"permissions_data": {
20+
"read": {
21+
"users": [
22+
"03887347926440898356",
23+
"04774006893477068632",
24+
"09147371668407854156",
25+
"13662041828528429192",
26+
"18298851591250030956"
27+
],
28+
"groups": [
29+
"10619079449796831495"
30+
]
2631
},
27-
{
28-
"id": "10619079449796831495",
29-
"displayName": "fuse-team",
30-
"type": "group",
31-
"kind": "drive#permission",
32-
"emailAddress": "[email protected]",
33-
"role": "writer",
34-
"deleted": false
32+
"update": {
33+
"users": [
34+
"03887347926440898356",
35+
"04774006893477068632",
36+
"09147371668407854156",
37+
"13662041828528429192",
38+
"18298851591250030956"
39+
],
40+
"groups": [
41+
"10619079449796831495"
42+
]
3543
},
36-
{
37-
"id": "03887347926440898356",
38-
"displayName": "michal.martyniak",
39-
"type": "user",
40-
"kind": "drive#permission",
41-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWWmJCSvduWur55hl36IxwKs5FJ1FWMoK6KtFlNUHDBU-McvlI=s64",
42-
"emailAddress": "[email protected]",
43-
"role": "writer",
44-
"deleted": false,
45-
"pendingOwner": false
46-
},
47-
{
48-
"id": "13662041828528429192",
49-
"displayName": "rob",
50-
"type": "user",
51-
"kind": "drive#permission",
52-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjV31Wb5kVmEk66cog1KN-N_twpHHoDttcCQ9pRvE1cz1-FLHQ=s64",
53-
"emailAddress": "[email protected]",
54-
"role": "writer",
55-
"deleted": false,
56-
"pendingOwner": false
57-
},
58-
{
59-
"id": "18298851591250030956",
60-
"displayName": "[email protected]",
61-
"type": "user",
62-
"kind": "drive#permission",
63-
"photoLink": "https://lh3.googleusercontent.com/a/ACg8ocJok2KRwwYvrEDkeZVCYosHOMoa52GZa2qIIC1jScCRoFLHaQ=s64",
64-
"emailAddress": "[email protected]",
65-
"role": "writer",
66-
"deleted": false,
67-
"pendingOwner": false
68-
},
69-
{
70-
"id": "04774006893477068632",
71-
"displayName": "ryan",
72-
"type": "user",
73-
"kind": "drive#permission",
74-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjXeWpu7QcZuYqIl3p1mwqzS8XGFJ4RqA3Xjljfkm1DcFZ9M7A=s64",
75-
"emailAddress": "[email protected]",
76-
"role": "writer",
77-
"deleted": false,
78-
"pendingOwner": false
79-
},
80-
{
81-
"id": "09147371668407854156",
82-
"displayName": "roman",
83-
"type": "user",
84-
"kind": "drive#permission",
85-
"photoLink": "https://lh3.googleusercontent.com/a-/ALV-UjWoGrFCgXcF6CtiBIBLnAfM68qUnQaJOcgvg3qzfQ3W8Ch6dA=s64",
86-
"emailAddress": "[email protected]",
87-
"role": "owner",
88-
"deleted": false,
89-
"pendingOwner": false
44+
"delete": {
45+
"users": [
46+
"09147371668407854156"
47+
],
48+
"groups": []
9049
}
91-
],
50+
},
9251
"filesize_bytes": 36602
9352
}
9453
}

0 commit comments

Comments
 (0)