[GROWENG-124][GROWENG-126] Adding yugabytedb as connector in

unstructured.io

Summary:
* Adding tests for yugabytedb connector
* Added set-up for yugabytedb as a source and destination connector
* Added expected results for yugabytedb as source, destination and
stager

Author: gargsans-yb

test/integration/connectors/env_setup/sql/yugabytedb/destination/docker-compose.yaml

@@ -0,0 +1,17 @@
+services:
+  yugabytedb:
+    image: yugabytedb/yugabyte:latest
+    restart: always
+    container_name: yugabytedb_dest
+    command: bin/yugabyted start --daemon=false --initial_scripts_dir=/home/yugabyte/init_scripts
+    ports:
+     - 5433:5433
+    volumes:
+      - ./yugabytedb-schema.sql:/home/yugabyte/init_scripts/init.sql
+    healthcheck:
+      test: ["CMD-SHELL", "bin/ysqlsh -h localhost -U yugabyte -d yugabyte -c 'SELECT 1'"]
+      interval: 10s
+      timeout: 60s
+      retries: 10
+      start_period: 20s
+

test/integration/connectors/env_setup/sql/yugabytedb/destination/yugabytedb-schema.sql

@@ -0,0 +1,48 @@
+-- Enable pgvector extension (YugabyteDB has native support)
+CREATE EXTENSION IF NOT EXISTS vector;
+
+-- Create the elements table with vector support in yugabyte database
+CREATE TABLE elements (
+    id UUID PRIMARY KEY,
+    record_id VARCHAR,
+    element_id VARCHAR,
+    text TEXT,
+    embeddings vector(384),
+    type VARCHAR,
+    system VARCHAR,
+    layout_width DECIMAL,
+    layout_height DECIMAL,
+    points TEXT,
+    url TEXT,
+    version VARCHAR,
+    date_created TIMESTAMPTZ,
+    date_modified TIMESTAMPTZ,
+    date_processed TIMESTAMPTZ,
+    permissions_data TEXT,
+    record_locator TEXT,
+    category_depth INTEGER,
+    parent_id VARCHAR,
+    attached_filename VARCHAR,
+    filetype VARCHAR,
+    last_modified TIMESTAMPTZ,
+    file_directory VARCHAR,
+    filename VARCHAR,
+    languages VARCHAR [],
+    page_number VARCHAR,
+    links TEXT,
+    page_name VARCHAR,
+    link_urls VARCHAR [],
+    link_texts VARCHAR [],
+    sent_from VARCHAR [],
+    sent_to VARCHAR [],
+    subject VARCHAR,
+    section VARCHAR,
+    header_footer_type VARCHAR,
+    emphasized_text_contents VARCHAR [],
+    emphasized_text_tags VARCHAR [],
+    text_as_html TEXT,
+    regex_metadata TEXT,
+    detection_class_prob DECIMAL
+);
+
+

test/integration/connectors/env_setup/sql/yugabytedb/source/docker-compose.yaml

@@ -0,0 +1,17 @@
+services:
+  yugabytedb:
+    image: yugabytedb/yugabyte:latest
+    restart: always
+    container_name: yugabytedb_src
+    command: bin/yugabyted start --daemon=false --initial_scripts_dir=/home/yugabyte/init_scripts
+    ports:
+     - 5433:5433
+    volumes:
+      - ./yugabytedb-schema.sql:/home/yugabyte/init_scripts/init.sql
+    healthcheck:
+      test: ["CMD-SHELL", "bin/ysqlsh -h localhost -U yugabyte -d yugabyte -c 'SELECT 1'"]
+      interval: 10s
+      timeout: 60s
+      retries: 10
+      start_period: 20s
+

test/integration/connectors/env_setup/sql/yugabytedb/source/yugabytedb-schema.sql

@@ -0,0 +1,8 @@
+-- Create the cars table in the yugabyte database
+CREATE TABLE cars (
+    car_id SERIAL PRIMARY KEY,
+    brand TEXT NOT NULL,
+    price INTEGER NOT NULL
+);
+
+

test/integration/connectors/expected_results/yugabytedb/directory_structure.json

@@ -0,0 +1,14 @@
+{
+  "directory_structure": [
+    "cars-1-5fb93ce5.csv",
+    "cars-10-5fb93ce5.csv",
+    "cars-2-5fb93ce5.csv",
+    "cars-3-5fb93ce5.csv",
+    "cars-4-5fb93ce5.csv",
+    "cars-5-5fb93ce5.csv",
+    "cars-6-5fb93ce5.csv",
+    "cars-7-5fb93ce5.csv",
+    "cars-8-5fb93ce5.csv",
+    "cars-9-5fb93ce5.csv"
+  ]
+}

test/integration/connectors/expected_results/yugabytedb/downloads/cars-1-5fb93ce5.csv

@@ -0,0 +1,2 @@
+car_id,brand
+1,brand_0

test/integration/connectors/expected_results/yugabytedb/downloads/cars-10-5fb93ce5.csv

@@ -0,0 +1,2 @@
+car_id,brand
+10,brand_9

test/integration/connectors/expected_results/yugabytedb/downloads/cars-2-5fb93ce5.csv

@@ -0,0 +1,2 @@
+car_id,brand
+2,brand_1

test/integration/connectors/expected_results/yugabytedb/downloads/cars-3-5fb93ce5.csv

@@ -0,0 +1,2 @@
+car_id,brand
+3,brand_2

test/integration/connectors/expected_results/yugabytedb/downloads/cars-4-5fb93ce5.csv

@@ -0,0 +1,2 @@
+car_id,brand
+4,brand_3