Skip to content

Commit f0a92ff

Browse files
vischvisch
committed
Waitlist
1 parent 2336318 commit f0a92ff

File tree

4 files changed

+342
-5
lines changed

4 files changed

+342
-5
lines changed

‎meltano.yml‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
version: 1
22
send_anonymous_usage_stats: true
3-
project_id: "tap-clerk"
3+
project_id: tap-clerk
44
default_environment: test
55
environments:
66
- name: test
77
plugins:
88
extractors:
9-
- name: "tap-clerk"
10-
namespace: "tap_clerk"
9+
- name: tap-clerk
10+
namespace: tap_clerk
1111
pip_url: -e .
1212
capabilities:
1313
- state
@@ -19,10 +19,20 @@ plugins:
1919
# TODO: Declare settings and their types here:
2020
settings:
2121
- name: auth_token
22-
kind: password
22+
kind: string
2323
description: Auth Token Needed
2424
sensitive: true
25+
select:
26+
- waitlist.*
2527
loaders:
2628
- name: target-jsonl
2729
variant: andyh1203
2830
pip_url: target-jsonl
31+
- name: target-postgres
32+
variant: meltanolabs
33+
pip_url: meltanolabs-target-postgres
34+
config:
35+
host: localhost
36+
port: 5432
37+
user: postgres
38+
database: postgres

‎plugins/loaders/target-postgres--meltanolabs.lock‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
{
2+
"plugin_type": "loaders",
3+
"name": "target-postgres",
4+
"namespace": "target_postgres",
5+
"variant": "meltanolabs",
6+
"label": "Postgres",
7+
"docs": "https://hub.meltano.com/loaders/target-postgres--meltanolabs",
8+
"repo": "https://github.com/MeltanoLabs/target-postgres",
9+
"pip_url": "meltanolabs-target-postgres",
10+
"executable": "target-postgres",
11+
"description": "PostgreSQL database loader",
12+
"logo_url": "https://hub.meltano.com/assets/logos/loaders/postgres.png",
13+
"capabilities": [
14+
"about",
15+
"activate-version",
16+
"hard-delete",
17+
"schema-flattening",
18+
"stream-maps"
19+
],
20+
"settings_group_validation": [
21+
[]
22+
],
23+
"settings": [
24+
{
25+
"name": "activate_version",
26+
"kind": "boolean",
27+
"value": true,
28+
"label": "Activate Version",
29+
"description": "If set to false, the tap will ignore activate version messages. If set to true, add_record_metadata must be set to true as well."
30+
},
31+
{
32+
"name": "add_record_metadata",
33+
"kind": "boolean",
34+
"value": true,
35+
"label": "Add Record Metadata",
36+
"description": "Note that this must be enabled for activate_version to work!This adds _sdc_extracted_at, _sdc_batched_at, and more to every table. See https://sdk.meltano.com/en/latest/implementation/record_metadata.html for more information."
37+
},
38+
{
39+
"name": "batch_size_rows",
40+
"kind": "integer",
41+
"label": "Batch Size Rows",
42+
"description": "Maximum number of rows in each batch."
43+
},
44+
{
45+
"name": "database",
46+
"kind": "string",
47+
"label": "Database",
48+
"description": "Database name."
49+
},
50+
{
51+
"name": "default_target_schema",
52+
"kind": "string",
53+
"value": "$MELTANO_EXTRACT__LOAD_SCHEMA",
54+
"label": "Default Target Schema",
55+
"description": "Postgres schema to send data to, example: tap-clickup"
56+
},
57+
{
58+
"name": "dialect+driver",
59+
"kind": "string",
60+
"value": "postgresql+psycopg",
61+
"label": "Dialect+Driver",
62+
"description": "DEPRECATED. Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone."
63+
},
64+
{
65+
"name": "faker_config.locale",
66+
"kind": "array",
67+
"label": "Faker Locale",
68+
"description": "One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization"
69+
},
70+
{
71+
"name": "faker_config.seed",
72+
"kind": "string",
73+
"label": "Faker Seed",
74+
"description": "Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator"
75+
},
76+
{
77+
"name": "flattening_enabled",
78+
"kind": "boolean",
79+
"label": "Enable Schema Flattening",
80+
"description": "'True' to enable schema flattening and automatically expand nested properties."
81+
},
82+
{
83+
"name": "flattening_max_depth",
84+
"kind": "integer",
85+
"label": "Max Flattening Depth",
86+
"description": "The max depth to flatten schemas."
87+
},
88+
{
89+
"name": "hard_delete",
90+
"kind": "boolean",
91+
"value": false,
92+
"label": "Hard Delete",
93+
"description": "When activate version is sent from a tap this specefies if we should delete the records that don't match, or mark them with a date in the `_sdc_deleted_at` column. This config option is ignored if `activate_version` is set to false."
94+
},
95+
{
96+
"name": "host",
97+
"kind": "string",
98+
"label": "Host",
99+
"description": "Hostname for postgres instance."
100+
},
101+
{
102+
"name": "interpret_content_encoding",
103+
"kind": "boolean",
104+
"value": false,
105+
"label": "Interpret Content Encoding",
106+
"description": "If set to true, the target will interpret the content encoding of the schema to determine how to store the data. Using this option may result in a more efficient storage of the data but may also result in an error if the data is not encoded as expected."
107+
},
108+
{
109+
"name": "load_method",
110+
"kind": "options",
111+
"value": "append-only",
112+
"label": "Load Method",
113+
"description": "The method to use when loading data into the destination. `append-only` will always write all input records whether that records already exists or not. `upsert` will update existing records and insert new records. `overwrite` will delete all existing records and insert all input records.",
114+
"options": [
115+
{
116+
"label": "Append Only",
117+
"value": "append-only"
118+
},
119+
{
120+
"label": "Upsert",
121+
"value": "upsert"
122+
},
123+
{
124+
"label": "Overwrite",
125+
"value": "overwrite"
126+
}
127+
]
128+
},
129+
{
130+
"name": "password",
131+
"kind": "string",
132+
"label": "Password",
133+
"description": "Password used to authenticate.",
134+
"sensitive": true
135+
},
136+
{
137+
"name": "port",
138+
"kind": "integer",
139+
"value": 5432,
140+
"label": "Port",
141+
"description": "The port on which postgres is awaiting connections."
142+
},
143+
{
144+
"name": "process_activate_version_messages",
145+
"kind": "boolean",
146+
"value": true,
147+
"label": "Process `ACTIVATE_VERSION` messages",
148+
"description": "Whether to process `ACTIVATE_VERSION` messages."
149+
},
150+
{
151+
"name": "sanitize_null_text_characters",
152+
"kind": "boolean",
153+
"value": false,
154+
"label": "Sanitize Null Text Characters",
155+
"description": "If set to true, the target will sanitize null characters in char/text/varchar fields, as they are not supported by Postgres. See [postgres documentation](https://www.postgresql.org/docs/current/functions-string.html) for more information about chr(0) not being supported."
156+
},
157+
{
158+
"name": "sqlalchemy_url",
159+
"kind": "string",
160+
"label": "SQLAlchemy URL",
161+
"description": "DEPRECATED. SQLAlchemy connection string. This will override using host, user, password, port, dialect, and all ssl settings. Note that you must escape password special characters properly. See https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords"
162+
},
163+
{
164+
"name": "ssh_tunnel.enable",
165+
"kind": "boolean",
166+
"value": false,
167+
"label": "SSH Tunnel Enable",
168+
"description": "Enable an ssh tunnel (also known as bastion host), see the other ssh_tunnel.* properties for more details"
169+
},
170+
{
171+
"name": "ssh_tunnel.host",
172+
"kind": "string",
173+
"label": "SSH Tunnel Host",
174+
"description": "Host of the bastion host, this is the host we'll connect to via ssh"
175+
},
176+
{
177+
"name": "ssh_tunnel.port",
178+
"kind": "integer",
179+
"value": 22,
180+
"label": "SSH Tunnel Port",
181+
"description": "Port to connect to bastion host"
182+
},
183+
{
184+
"name": "ssh_tunnel.private_key",
185+
"kind": "string",
186+
"label": "SSH Tunnel Private Key",
187+
"description": "Private Key for authentication to the bastion host",
188+
"sensitive": true
189+
},
190+
{
191+
"name": "ssh_tunnel.private_key_password",
192+
"kind": "string",
193+
"label": "SSH Tunnel Private Key Password",
194+
"description": "Private Key Password, leave None if no password is set",
195+
"sensitive": true
196+
},
197+
{
198+
"name": "ssh_tunnel.username",
199+
"kind": "string",
200+
"label": "SSH Tunnel Username",
201+
"description": "Username to connect to bastion host"
202+
},
203+
{
204+
"name": "ssl_certificate_authority",
205+
"kind": "string",
206+
"value": "~/.postgresql/root.crl",
207+
"label": "SSL Certificate Authority",
208+
"description": "The certificate authority that should be used to verify the server's identity. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate."
209+
},
210+
{
211+
"name": "ssl_client_certificate",
212+
"kind": "string",
213+
"value": "~/.postgresql/postgresql.crt",
214+
"label": "SSL Client Certificate",
215+
"description": "The certificate that should be used to verify your identity to the server. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate."
216+
},
217+
{
218+
"name": "ssl_client_certificate_enable",
219+
"kind": "boolean",
220+
"value": false,
221+
"label": "SSL Client Certificate Enable",
222+
"description": "Whether or not to provide client-side certificates as a method of authentication to the server. Use ssl_client_certificate and ssl_client_private_key for further customization. To use SSL to verify the server's identity, use ssl_enable instead."
223+
},
224+
{
225+
"name": "ssl_client_private_key",
226+
"kind": "string",
227+
"value": "~/.postgresql/postgresql.key",
228+
"label": "SSL Client Private Key",
229+
"description": "The private key for the certificate you provided. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate.",
230+
"sensitive": true
231+
},
232+
{
233+
"name": "ssl_enable",
234+
"kind": "boolean",
235+
"value": false,
236+
"label": "SSL Enable",
237+
"description": "Whether or not to use ssl to verify the server's identity. Use ssl_certificate_authority and ssl_mode for further customization. To use a client certificate to authenticate yourself to the server, use ssl_client_certificate_enable instead."
238+
},
239+
{
240+
"name": "ssl_mode",
241+
"kind": "string",
242+
"value": "verify-full",
243+
"label": "SSL Mode",
244+
"description": "SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) for more information. Must be one of disable, allow, prefer, require, verify-ca, or verify-full."
245+
},
246+
{
247+
"name": "ssl_storage_directory",
248+
"kind": "string",
249+
"value": ".secrets",
250+
"label": "SSL Storage Directory",
251+
"description": "The folder in which to store SSL certificates provided as raw values. When a certificate/key is provided as a raw value instead of as a filepath, it must be written to a file before it can be used. This configuration option determines where that file is created."
252+
},
253+
{
254+
"name": "stream_map_config",
255+
"kind": "object",
256+
"label": "User Stream Map Configuration",
257+
"description": "User-defined config values to be used within map expressions."
258+
},
259+
{
260+
"name": "stream_maps",
261+
"kind": "object",
262+
"label": "Stream Maps",
263+
"description": "Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)."
264+
},
265+
{
266+
"name": "use_copy",
267+
"kind": "boolean",
268+
"value": false,
269+
"label": "Use COPY",
270+
"description": "Use the COPY command to insert data. This is usually faster than INSERT statements. This option is only available for the postgresql+psycopg dialect+driver."
271+
},
272+
{
273+
"name": "user",
274+
"kind": "string",
275+
"label": "User",
276+
"description": "User name used to authenticate."
277+
},
278+
{
279+
"name": "validate_records",
280+
"kind": "boolean",
281+
"value": true,
282+
"label": "Validate Records",
283+
"description": "Whether to validate the schema of the incoming streams."
284+
}
285+
],
286+
"dialect": "postgres",
287+
"target_schema": "$TARGET_POSTGRES_SCHEMA"
288+
}

‎tap_clerk/streams.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,4 +224,42 @@ def get_url_params(self, context: dict | None, next_page_token: t.Any | None) ->
224224
if self.replication_key:
225225
params["order_by"] = f'+{self.replication_key}'
226226
self.logger.info(f"QUERY PARAMS: {params}")
227-
return params
227+
return params
228+
229+
class WaitlistEntriesStream(ClerkStream):
230+
"""Waitlist entries stream class."""
231+
name = "waitlist"
232+
path = "/waitlist_entries"
233+
primary_keys: t.ClassVar[list[str]] = ["id"]
234+
replication_key = None
235+
schema = th.PropertiesList(
236+
th.Property("object", th.StringType),
237+
th.Property("id", th.StringType),
238+
th.Property("email_address", th.StringType),
239+
th.Property("status", th.StringType),
240+
th.Property("created_at", th.IntegerType),
241+
th.Property("updated_at", th.IntegerType),
242+
th.Property("invitation", th.ObjectType(
243+
th.Property("object", th.StringType),
244+
th.Property("id", th.StringType),
245+
th.Property("email_address", th.StringType),
246+
th.Property("public_metadata", th.ObjectType(additional_properties=True)),
247+
th.Property("revoked", th.BooleanType),
248+
th.Property("status", th.StringType),
249+
th.Property("url", th.StringType),
250+
th.Property("expires_at", th.IntegerType),
251+
th.Property("created_at", th.IntegerType),
252+
th.Property("updated_at", th.IntegerType)
253+
))
254+
).to_dict()
255+
256+
def get_url_params(self, context: dict | None, next_page_token: t.Any | None) -> dict[str, t.Any]:
257+
params: dict = {"limit": self.API_LIMIT_PAGE_SIZE}
258+
if next_page_token:
259+
params["offset"] = next_page_token
260+
if self.replication_key:
261+
params["order_by"] = f'+{self.replication_key}'
262+
return params
263+
264+
def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
265+
yield from extract_jsonpath("$.data[*]", input=response.json())

‎tap_clerk/tap.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def discover_streams(self) -> list[streams.ClerkStream]:
2323
streams.OrganizationsStream(self),
2424
streams.OrganizationMembershipStream(self),
2525
streams.UsersStream(self),
26+
streams.WaitlistEntriesStream(self),
2627
]
2728

2829
if __name__ == "__main__":

0 commit comments

Comments
 (0)