feat: Migrate Notion source to Connector V2 structure

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Enhancements
 
+* **Migrate Notion Source Connector to V2**
 * **Migrate Vectara Destination Connector to v2**
 * **Added Redis destination connector**
 * **Improved Milvus error handling**

test/integration/connectors/expected_results/notion_database/directory_structure.json

@@ -0,0 +1,5 @@
+{
+  "directory_structure": [
+    "1722c3765a0a8082b382ebc2c62d3f4c.html"
+  ]
+}

test/integration/connectors/expected_results/notion_database/downloads/1722c3765a0a8082b382ebc2c62d3f4c.html

@@ -0,0 +1,330 @@
+<table>
+  <tr>
+    <th>
+      Author
+    </th>
+    <th>
+      Email
+    </th>
+    <th>
+      Formula
+    </th>
+    <th>
+      ID
+    </th>
+    <th>
+      Item
+    </th>
+    <th>
+      Phone
+    </th>
+    <th>
+      Priority Level
+    </th>
+    <th>
+      Publication Date
+    </th>
+    <th>
+      Status
+    </th>
+    <th>
+      Tag
+    </th>
+    <th>
+      URL
+    </th>
+    <th>
+      Views
+    </th>
+  </tr>
+  <tr>
+    <td>
+      <div>
+        <span>
+          <a href='https://lh3.googleusercontent.com/a/AATXAJyU_JkTO5JUkbJ8l_RuIHq4K3KdPghBV0r9U_GU=s100'>
+            Brian Raymond
+          </a>
+        </span>
+      </div>
+    </td>
+    <td>
+      <div>
+        [email protected]
+      </div>
+    </td>
+    <td>
+      <div>
+        12
+      </div>
+    </td>
+    <td>
+      <div>
+        None-4
+      </div>
+    </td>
+    <td>
+      <div>
+        test-page4-in-database
+      </div>
+    </td>
+    <td>
+      <div>
+        1234567890
+      </div>
+    </td>
+    <td>
+      <div style='color: gray'>
+        High
+      </div>
+    </td>
+    <td>
+      <div>
+        2025-01-31
+      </div>
+    </td>
+    <td>
+      <div>
+        Not started
+      </div>
+    </td>
+    <td>
+      <div>
+        <span style='color: purple'>
+          V1
+        </span>
+        <span style='color: yellow'>
+          V5
+        </span>
+        <span style='color: gray'>
+          V7
+        </span>
+      </div>
+    </td>
+    <td>
+      <a href='https://abcde.com'>
+        https://abcde.com
+      </a>
+    </td>
+    <td>
+      <div>
+        6
+      </div>
+    </td>
+  </tr>
+  <tr>
+    <td>
+      <div>
+        <span>
+          <a href='https://lh3.googleusercontent.com/a/AATXAJyU_JkTO5JUkbJ8l_RuIHq4K3KdPghBV0r9U_GU=s100'>
+            Brian Raymond
+          </a>
+        </span>
+      </div>
+    </td>
+    <td>
+      <div>
+        [email protected]
+      </div>
+    </td>
+    <td>
+      <div>
+        90
+      </div>
+    </td>
+    <td>
+      <div>
+        None-3
+      </div>
+    </td>
+    <td>
+      <div>
+        test-page3-in-database
+      </div>
+    </td>
+    <td>
+      <div>
+        1234567890
+      </div>
+    </td>
+    <td>
+      <div style='color: pink'>
+        Medium
+      </div>
+    </td>
+    <td>
+      <div>
+        2025-01-06
+      </div>
+    </td>
+    <td>
+      <div style='color: orange'>
+        In Review
+      </div>
+    </td>
+    <td>
+      <div>
+        <span style='color: yellow'>
+          V5
+        </span>
+        <span>
+          V6
+        </span>
+      </div>
+    </td>
+    <td>
+      <a href='https://abcde.com'>
+        https://abcde.com
+      </a>
+    </td>
+    <td>
+      <div>
+        45
+      </div>
+    </td>
+  </tr>
+  <tr>
+    <td>
+      <div>
+        <span>
+          <a href='https://lh3.googleusercontent.com/a/AATXAJyU_JkTO5JUkbJ8l_RuIHq4K3KdPghBV0r9U_GU=s100'>
+            Brian Raymond
+          </a>
+        </span>
+      </div>
+    </td>
+    <td>
+      <div>
+        [email protected]
+      </div>
+    </td>
+    <td>
+      <div>
+        46
+      </div>
+    </td>
+    <td>
+      <div>
+        None-2
+      </div>
+    </td>
+    <td>
+      <div>
+        test-page2-in-database
+      </div>
+    </td>
+    <td>
+      <div>
+        1234567890
+      </div>
+    </td>
+    <td>
+      <div style='color: blue'>
+        Low
+      </div>
+    </td>
+    <td>
+      <div>
+        2025-01-04
+      </div>
+    </td>
+    <td>
+      <div style='color: green'>
+        Done
+      </div>
+    </td>
+    <td>
+      <div>
+        <span style='color: purple'>
+          V1
+        </span>
+        <span style='color: orange'>
+          V2
+        </span>
+        <span style='color: brown'>
+          V4
+        </span>
+      </div>
+    </td>
+    <td>
+      <a href='https://abcde.com'>
+        https://abcde.com
+      </a>
+    </td>
+    <td>
+      <div>
+        23
+      </div>
+    </td>
+  </tr>
+  <tr>
+    <td>
+      <div>
+        <span>
+          <a href='https://lh3.googleusercontent.com/a/AATXAJyU_JkTO5JUkbJ8l_RuIHq4K3KdPghBV0r9U_GU=s100'>
+            Brian Raymond
+          </a>
+        </span>
+      </div>
+    </td>
+    <td>
+      <div>
+        [email protected]
+      </div>
+    </td>
+    <td>
+      <div>
+        4
+      </div>
+    </td>
+    <td>
+      <div>
+        None-1
+      </div>
+    </td>
+    <td>
+      <div>
+        test-page1-in-datab
+      </div>
+    </td>
+    <td>
+      <div>
+        1234567890
+      </div>
+    </td>
+    <td>
+      <div style='color: gray'>
+        High
+      </div>
+    </td>
+    <td>
+      <div>
+        2024-12-01
+      </div>
+    </td>
+    <td>
+      <div style='color: blue'>
+        In progress
+      </div>
+    </td>
+    <td>
+      <div>
+        <span style='color: purple'>
+          V1
+        </span>
+        <span style='color: green'>
+          V3
+        </span>
+      </div>
+    </td>
+    <td>
+      <a href='https://abcde.com'>
+        https://abcde.com
+      </a>
+    </td>
+    <td>
+      <div>
+        2
+      </div>
+    </td>
+  </tr>
+</table>

test/integration/connectors/expected_results/notion_database/file_data/1722c3765a0a8082b382ebc2c62d3f4c.json

@@ -0,0 +1,39 @@
+{
+  "identifier": "1722c3765a0a8082b382ebc2c62d3f4c",
+  "connector_type": "notion",
+  "source_identifiers": {
+    "filename": "1722c3765a0a8082b382ebc2c62d3f4c.html",
+    "fullpath": "1722c3765a0a8082b382ebc2c62d3f4c.html",
+    "rel_path": "1722c3765a0a8082b382ebc2c62d3f4c.html"
+  },
+  "metadata": {
+    "url": null,
+    "version": null,
+    "record_locator": {
+      "database_id": "1722c3765a0a8082b382ebc2c62d3f4c"
+    },
+    "date_created": "2025-01-05T18:34:00.000Z",
+    "date_modified": "2025-01-07T19:15:00.000Z",
+    "date_processed": "1736277913.3980532",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "created_by": {
+      "id": "118d872b-594c-8171-b46f-00020d10d8b2",
+      "object": "user"
+    },
+    "last_edited_by": {
+      "id": "118d872b-594c-8171-b46f-00020d10d8b2",
+      "object": "user"
+    },
+    "parent": {
+      "type": "workspace",
+      "workspace": true
+    },
+    "url": "https://www.notion.so/1722c3765a0a8082b382ebc2c62d3f4c"
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmpxu906ary/1722c3765a0a8082b382ebc2c62d3f4c.html",
+  "display_name": null
+}

test/integration/connectors/expected_results/notion_page/directory_structure.json

@@ -0,0 +1,5 @@
+{
+  "directory_structure": [
+    "1572c3765a0a806299f0dd6999f9e4c7.html"
+  ]
+}