From 18cc98e6b5164b3992a25d0608535e8d110d25ad Mon Sep 17 00:00:00 2001 From: John Duff Date: Wed, 30 Apr 2025 10:44:29 -0400 Subject: [PATCH 1/3] Refactor schma inference to decouple from scraper --- src/infer-schema.ts | 649 +++++++++++++++++++++++++++++++++++++++++++ src/scrape.ts | 658 +------------------------------------------- 2 files changed, 662 insertions(+), 645 deletions(-) create mode 100644 src/infer-schema.ts diff --git a/src/infer-schema.ts b/src/infer-schema.ts new file mode 100644 index 0000000..00ae60a --- /dev/null +++ b/src/infer-schema.ts @@ -0,0 +1,649 @@ +import { getVocabulary, schemaWalk } from "@cloudflare/json-schema-walker"; +import { inferSchema } from "@jsonhero/schema-infer"; + +import { cloneDeep, cloneDeepWith, isEqual, merge, pick } from "lodash"; +import { configure } from "safe-stable-stringify"; +const stringify = configure({ deterministic: true }); + +export const startVersion = "2024-04"; + +export type Error = { message: string; path: string }; + +export const inferSchemaFromExamplePayload = ( + examplePayload: Record, + metadata: { name: string } +): { schema: any; warnings: number; errors: Error[] } => { + const inference = inferSchema(examplePayload); + + // build a copy of the payload and apply overrides based on the webhook name + for (const [matcher, override] of manualExamples) { + if (matcher.test(metadata.name)) { + const overridesPayload = cloneDeep(examplePayload); + const maskedOverride = pick(override, Object.keys(examplePayload)); + merge(overridesPayload, maskedOverride); + inference.infer(overridesPayload); + } + } + + const schema = { + $schema: "https://json-schema.org/draft/2020-12/schema", + ...(inference.toJSONSchema() as any), + }; + + for (const override of overrides) { + if (override.topics.includes(metadata.name) && (!override.versions || override.versions.includes(startVersion))) { + for (const [key, schemaOverride] of Object.entries(override.schema)) { + schema.properties[key] = schemaOverride; + } + } + } + + const sortedSchema = getDeterministicObject(schema); + + let warnings = 0; + let errors: Error[] = []; + + schemaWalk( + sortedSchema, + (subschema, path, _parent, parentPath) => { + if (isEqual(subschema, { type: "null" })) { + warnings += 1; + const fullPath = [...parentPath, ...path].join("."); + if (unknownPaths.some(([pattern, paths]) => pattern.test(metadata.name) && paths.includes(fullPath))) { + // we know this path is always null, so don't error + } else { + errors.push({ + message: "null type found in final schema", + path: fullPath, + }); + } + } + }, + () => {}, + getVocabulary(sortedSchema) + ); + + return { schema: sortedSchema, warnings, errors }; +}; + +const getDeterministicObject = (obj: Record): Record => { + const stableString = stringify(sortStringArrays(obj)); + return JSON.parse(stableString!); +}; + +const sortStringArrays = (obj: Record) => { + return cloneDeepWith(obj, (value) => { + if (Array.isArray(value) && value.every((item) => typeof item === "string")) { + return value.sort(); + } + }); +}; + +// paths that we can't find any source data for at all, so we don't know what type they should be +const unknownPaths: [topicPattern: RegExp, paths: string[]][] = [ + [ + /checkouts\/.+/, + [ + "properties.line_items.items.properties.unit_price_measurement.properties.measured_type", + "properties.line_items.items.properties.unit_price_measurement.properties.quantity_value", + "properties.line_items.items.properties.unit_price_measurement.properties.quantity_unit", + "properties.line_items.items.properties.unit_price_measurement.properties.reference_value", + "properties.line_items.items.properties.unit_price_measurement.properties.reference_unit", + "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_name", + "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_type", + "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_code", + "properties.line_items.items.properties.user_id", + "properties.line_items.items.properties.compare_at_price", + "properties.gateway", + "properties.shipping_lines.items.properties.delivery_category", + "properties.shipping_lines.items.properties.validation_context", + "properties.shipping_lines.items.properties.requested_fulfillment_service_id", + "properties.shipping_lines.items.properties.tax_lines.items.properties.identifier", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_name", + "properties.shipping_lines.items.properties.tax_lines.items.properties.tax_api_client_id", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_exempt_amount", + "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_source", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_code", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_type", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_taxable_amount", + "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_type", + "properties.shipping_lines.items.properties.tax_lines.items.properties.tax_type", + "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_id", + "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_non_taxable_amount", + "properties.shipping_lines.items.properties.custom_tax_lines", + "properties.shipping_lines.items.properties.estimated_delivery_time_range", + "properties.line_items.items.properties.tax_lines.items.properties.tax_type", + "properties.line_items.items.properties.tax_lines.items.properties.identifier", + "properties.line_items.items.properties.discount_allocations.items.properties.id", + "properties.line_items.items.properties.discount_allocations.items.properties.created_at", + ], + ], + [ + /collection_listings\/.+/, + ["properties.collection_listing.properties.default_product_image", "properties.collection_listing.properties.image"], + ], + [/domains\/.+/, ["properties.localization.properties.country"]], + [/orders\/.+/, ["properties.client_details.properties.session_hash"]], +]; + +// example data we feed the schema infer-er for each topic to allow it to discover real types +export const manualExamples: [RegExp, Record][] = [ + [ + /.+/, + { + admin_graphql_api_id: "gid://shopify/Something/1234567890", + admin_graphql_api_job_id: "gid://shopify/Job/1234567890", + created_at: "2021-12-30T19:00:00-05:00", + updated_at: "2021-12-30T19:00:00-05:00", + address2: "Apt 123", + latitude: 10.1, + longitude: 10.1, + location_id: 111111, + }, + ], + [ + /(app|shop)\/.+/, + { + domain: "example.com", + source: "example source", + myshopify_domain: "example.myshopify.com", + google_apps_domain: "example.com", + google_apps_login_enabled: true, + password_enabled: true, + taxes_included: true, + tax_shipping: true, + iana_timezone: "America/New_York", + auto_configure_tax_inclusivity: true, + county_taxes: true, + }, + ], + [ + /bulk_operations\/.+/, + { + error_code: "SOME_ERROR_ENUM", + }, + ], + [ + /carts\/.+/, + { + note: "some cart note string", + }, + ], + [ + /(checkouts|orders)\/.+/, + { + gateway: "shopify_payments", + landing_site: "https://example.com", + note: "some order note", + referring_site: "https://example.com", + completed_at: "2021-12-30T19:00:00-05:00", + closed_at: "2021-12-30T19:00:00-05:00", + user_id: 11111111, + location_id: 22222222, + source_identifier: "some_source_identifier", + source_url: "https://example.com", + device_id: "some_device_id", + phone: "+1 (123) 456 7890", + sms_marketing_phone: "+1 (123) 456 7890", + customer_locale: "en", + source: "some_source", + total_duties: 10.11, + app_id: 12345, + browser_ip: "10.0.0.1", + cart_token: "some_cart_token", + checkout_id: 12345, + client_details: { + accept_language: "en-US,en;q=0.9", + browser_height: 800, + }, + confirmation_number: "some_confirmation_number", + current_total_additional_fees_set: { + shop_money: { + amount: "0.00", + currency_code: "USD", + }, + presentment_money: { + amount: "0.00", + currency_code: "USD", + }, + }, + current_total_duties_set: { + shop_money: { + amount: "0.00", + currency_code: "USD", + }, + presentment_money: { + amount: "0.00", + currency_code: "USD", + }, + }, + original_total_additional_fees_set: { + shop_money: { + amount: "0.00", + currency_code: "USD", + }, + presentment_money: { + amount: "0.00", + currency_code: "USD", + }, + }, + original_total_duties_set: { + shop_money: { + amount: "0.00", + currency_code: "USD", + }, + presentment_money: { + amount: "0.00", + currency_code: "USD", + }, + }, + checkout_token: "some_checkout_token", + landing_site_ref: "https://example.com", + merchant_of_record_app_id: 12345, + po_number: "some_po_number", + processed_at: "2021-12-30T19:00:00-05:00", + reference: "some_reference", + payment_terms: "some_payment_terms", + reservation_token: "some_reservation_token", + billing_address: { + address2: "suite 101", + latitude: 34.1, + longitude: 34.1, + }, + shipping_address: { + address2: "suite 101", + latitude: 34.1, + longitude: 34.1, + }, + customer: { + created_at: "2024-05-05T02:42:32+08:00", + marketing_opt_in_level: "single_opt_in", + note: "some string", + multipass_identifier: "hello", + }, + }, + ], + [ + /checkouts\/.+/, + { + line_items: [ + { + presentment_variant_title: "51 x 76 / Blanc / 1", + taxable: true, + variant_price: "69.00", + presentment_title: "Taie d'oreiller Pure Soie de Mûrier", + requires_shipping: true, + unit_price_measurement: { + measured_type: null, + quantity_unit: null, + quantity_value: null, + reference_unit: null, + reference_value: null, + }, + variant_title: "51 x 76 / Blanc / 1", + title: "Taie d'oreiller Pure Soie de Mûrier", + gift_card: false, + destination_location_id: 4042942054746, + compare_at_price: null, + key: "e0105d4efb1970501cf831566fa79752", + line_price: "69.00", + vendor: "Emily's pillow", + quantity: 1, + applied_discounts: [], + grams: 110, + properties: [ + { + name: "_isJust", + value: "true", + }, + ], + tax_lines: [ + { + reporting_jurisdiction_type: null, + reporting_non_taxable_amount: "0.00", + zone: null, + compare_at: 0.2, + reporting_taxable_amount: "57.50", + tax_api_client_id: null, + jurisdiction_source: "ActiveTax", + title: "FR TVA", + identifier: null, + jurisdiction_type: "COUNTRY", + reporting_jurisdiction_code: null, + source: "MerchantActiveTax", + reporting_exempt_amount: "0.00", + reporting_jurisdiction_name: null, + jurisdiction_id: "FR", + tax_type: null, + channel_liable: false, + price: "11.50", + rate: 0.2, + position: 1, + tax_calculation_price: "11.50", + }, + ], + sku: "EMIL-TO5176-22WH", + rank: 0, + user_id: null, + product_id: 4610455470216, + discount_allocations: [], + origin_location_id: 1743669821576, + fulfillment_service: "manual", + variant_id: 32516187029640, + price: "69.00", + }, + ], + }, + ], + [ + /collections\/.+/, + { + sort_order: "manual", + template_suffix: "some_template_suffix", + }, + ], + [ + /collection_listings\/.+/, + { + collection_listing: { + updated_at: "2021-12-30T19:00:00-05:00", + sort_order: 1, + }, + }, + ], + [ + /company_locations\/.+/, + { + buyer_experience_configuration: { pay_now_only: true }, + billing_address: { + address2: "suite 101", + }, + shipping_address: { + address2: "suite 101", + }, + }, + ], + [ + /customers\/.+/, + { + last_order_id: 12345, + multipass_identifier: "some_multipass_identifier", + last_order_name: "Foobar", + phone: "+1 (123) 456 7890", + sms_marketing_consent: false, + email_marketing_consent: false, + accepts_marketing_updated_at: "2021-12-30T19:00:00-05:00", + marketing_opt_in_level: "single_opt_in", + }, + ], + [ + /customer_account_settings\/.+/, + { + url: "https://example.com", + }, + ], + [ + /customer.+consent\/.+/, + { + phone: "+1 (123) 456 7890", + email_address: "test@test.com", + }, + ], + [ + /disputes\/.+/, + { + evidence_sent_on: "2021-12-30T19:00:00-05:00", + finalized_on: "2021-12-30T19:00:00-05:00", + }, + ], + [ + /draft_orders\/.+/, + { + invoice_sent_at: "2021-12-30T19:00:00-05:00", + order_id: 12345, + }, + ], + [ + /fulfillment_events\/.+/, + { + city: "Ottawa", + province: "ON", + zip: "K1P1J1", + address1: "150 Elgin St", + estimated_delivery_at: "2021-12-30T19:00:00-05:00", + }, + ], + [ + /fulfillment_orders\/.+/, + { + remaining_fulfillment_order: { + id: 5859333242902, + shop_id: 20978040854, + order_id: 4804938989590, + assigned_location_id: 67794436118, + request_status: "unsubmitted", + status: "open", + supported_actions: ["request_fulfillment", "hold", "move"], + destination: { + id: 5479404371990, + address1: "23 Hassall Street", + address2: "", + city: "Parramatta", + company: null, + country: "Australia", + email: "", + first_name: "Tyler", + last_name: "Kelleher", + phone: null, + province: "New South Wales", + zip: "2150", + }, + line_items: [ + { + id: 12675478814742, + shop_id: 20978040854, + fulfillment_order_id: 5859333242902, + quantity: 1, + line_item_id: 12553770336278, + inventory_item_id: 44276125368342, + fulfillable_quantity: 1, + variant_id: 42182036422678, + }, + ], + fulfill_at: "2022-10-13T13:00:00-04:00", + fulfill_by: null, + international_duties: { + incoterm: "DAP", + }, + fulfillment_holds: [], + delivery_method: { + id: 140816351254, + method_type: "shipping", + min_delivery_date_time: null, + max_delivery_date_time: null, + }, + assigned_location: { + address1: null, + address2: null, + city: null, + country_code: "CA", + location_id: 67794436118, + name: "test-created-via-api-2", + phone: null, + province: null, + zip: null, + }, + }, + }, + ], + [ + /fulfillments\/.+/, + { + service: "manual", + shipment_status: "confirmed", + + origin_address: { + first_name: "Steve", + address1: "123 Shipping Street", + phone: "555-555-SHIP", + city: "Shippington", + zip: "40003", + province: "Kentucky", + country: "United States", + last_name: "Shipper", + address2: null, + company: "Shipping Company", + latitude: null, + longitude: null, + name: "Steve Shipper", + country_code: "US", + province_code: "KY", + }, + }, + ], + [ + /inventory_items\/.+/, + { + cost: 10.11, + country_code_of_origin: "CA", + province_code_of_origin: "CA", + harmonized_system_code: "1234567890", + }, + ], + [ + /inventory_levels\/.+/, + { + available: 10, + }, + ], + [ + /order_transactions\/.+/, + { + message: "some message from the gateway", + user_id: 12345, + parent_id: 12345, + processed_at: "2021-12-30T19:00:00-05:00", + device_id: "some_device_id", + error_code: "SOME_ERROR_ENUM", + }, + ], + [ + /orders\/risk_assessment.+/, + { + provider_id: 12345, + provider_title: "whatever", + }, + ], + [ + /products\/.+/, + { + template_suffix: "something", + image: "gid://shopify/ProductImage/1234567890", + }, + ], + [ + /refunds\/.+/, + { + return: "unknown", + }, + ], + [ + /selling_plan_groups\/.+/, + { + app_id: 12345, + description: "some description", + position: 1, + }, + ], + [ + /subscription_billing_attempts\/.+/, + { + id: 12345, + error_message: "some error message", + error_code: "some error copde", + }, + ], + [ + /subscription_billing_cycle.+/, + { + contract_edit: "some contract edit", + }, + ], + [ + /tender_transactions\/.+/, + { + user_id: 12345, + processed_at: "2021-12-30T19:00:00-05:00", + payment_details: { something: "true" }, + }, + ], +]; + +const shippingAddress = { + type: "object", + properties: { + first_name: { + type: "string", + }, + address1: { + type: "string", + }, + phone: { + type: "string", + }, + city: { + type: "string", + }, + zip: { + type: "string", + }, + province: { + type: "string", + }, + country: { + type: "string", + }, + last_name: { + type: "string", + }, + address2: { + type: ["string", "null"], + }, + company: { + type: ["string", "null"], + }, + latitude: { + type: ["number", "null"], + }, + longitude: { + type: ["number", "null"], + }, + name: { + type: "string", + }, + country_code: { + type: "string", + }, + province_code: { + type: "string", + }, + }, +}; + +export const overrides: { topics: string[]; schema: any; versions?: string[] }[] = [ + { + topics: ["checkouts/create", "checkouts/update"], + schema: { + shipping_address: shippingAddress, + }, + }, + { + topics: ["order_transactions/create", "order_transactions/update"], + schema: { + receipt: { + type: "object", + additionalProperties: true, + }, + }, + }, +]; diff --git a/src/scrape.ts b/src/scrape.ts index f4daad9..164b3a7 100644 --- a/src/scrape.ts +++ b/src/scrape.ts @@ -1,19 +1,13 @@ -import { getVocabulary, schemaWalk } from "@cloudflare/json-schema-walker"; -import { inferSchema } from "@jsonhero/schema-infer"; -import chalk from "chalk"; import cheerio from "cheerio"; import fs from "fs-extra"; import { globby } from "globby"; import got from "got"; -import { cloneDeep, cloneDeepWith, isEqual, merge, pick, uniq } from "lodash"; +import { uniq } from "lodash"; +import chalk from "chalk"; import path from "path"; -import { configure } from "safe-stable-stringify"; import { getPackageRootDir } from "src"; import { startDecoding } from "./shopify.js"; - -const stringify = configure({ deterministic: true }); - -const startVersion = "2024-04"; +import { inferSchemaFromExamplePayload, manualExamples, startVersion } from "./infer-schema"; function assert(value: T | false | undefined | null, message?: string): T { if (!value) { @@ -72,72 +66,6 @@ const docsWebhooksPageForVersion = (version: string) => `https://shopify.dev/doc let warnings = 0; let errors = 0; -const sortStringArrays = (obj: Record) => { - return cloneDeepWith(obj, (value) => { - if (Array.isArray(value) && value.every((item) => typeof item === "string")) { - return value.sort(); - } - }); -}; - -const getDeterministicObject = (obj: Record): Record => { - const stableString = stringify(sortStringArrays(obj)); - return JSON.parse(stableString!); -}; - -const inferSchemaFromExamplePayload = (examplePayload: Record, metadata: { name: string }, version: string) => { - const inference = inferSchema(examplePayload); - - // build a copy of the payload and apply overrides based on the webhook name - for (const [matcher, override] of manualExamples) { - if (matcher.test(metadata.name)) { - const overridesPayload = cloneDeep(examplePayload); - const maskedOverride = pick(override, Object.keys(examplePayload)); - merge(overridesPayload, maskedOverride); - inference.infer(overridesPayload); - } - } - - const schema = { - $schema: "https://json-schema.org/draft/2020-12/schema", - ...(inference.toJSONSchema() as any), - }; - - for (const override of overrides) { - if (override.topics.includes(metadata.name) && (!override.versions || override.versions.includes(startVersion))) { - for (const [key, schemaOverride] of Object.entries(override.schema)) { - schema.properties[key] = schemaOverride; - } - } - } - - const sortedSchema = getDeterministicObject(schema); - - schemaWalk( - sortedSchema, - (subschema, path, _parent, parentPath) => { - if (isEqual(subschema, { type: "null" })) { - warnings += 1; - const fullPath = [...parentPath, ...path].join("."); - if (unknownPaths.some(([pattern, paths]) => pattern.test(metadata.name) && paths.includes(fullPath))) { - // we know this path is always null, so don't error - } else { - errors += 1; - console.error( - `${chalk.red("schema error")}: null type found in final schema for version ${chalk.blue(version)} for ${chalk.blue( - metadata.name - )} at path ${chalk.green(fullPath)}` - ); - } - } - }, - () => {}, - getVocabulary(sortedSchema) - ); - - return sortedSchema; -}; - const loadExemplars = async () => { const files = await globby(path.join(__dirname, "../exemplars/**/*.json")); for (const file of files) { @@ -179,7 +107,16 @@ const main = async () => { await fs.mkdir(path.dirname(metadataFile), { recursive: true }); await fs.writeFile(metadataFile, JSON.stringify(webhook, null, 2), "utf-8"); - const schema = inferSchemaFromExamplePayload(webhook.response, webhook, version); + const { schema, warnings: warningCount, errors: errorMessages } = inferSchemaFromExamplePayload(webhook.response, webhook); + warnings += warningCount; + for (const error of errorMessages) { + console.error( + `${chalk.red("schema error")}: ${error.message} for version ${chalk.blue(version)} for ${chalk.blue( + webhook.name + )} at path ${chalk.green(error.path)}` + ); + errors += 1; + } const schemaFile = path.join(rootDir, "schemas", version, webhook.name + ".json"); await fs.mkdir(path.dirname(schemaFile), { recursive: true }); @@ -196,572 +133,3 @@ const main = async () => { }; void main(); - -// paths that we can't find any source data for at all, so we don't know what type they should be -const unknownPaths: [topicPattern: RegExp, paths: string[]][] = [ - [ - /checkouts\/.+/, - [ - "properties.line_items.items.properties.unit_price_measurement.properties.measured_type", - "properties.line_items.items.properties.unit_price_measurement.properties.quantity_value", - "properties.line_items.items.properties.unit_price_measurement.properties.quantity_unit", - "properties.line_items.items.properties.unit_price_measurement.properties.reference_value", - "properties.line_items.items.properties.unit_price_measurement.properties.reference_unit", - "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_name", - "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_type", - "properties.line_items.items.properties.tax_lines.items.properties.reporting_jurisdiction_code", - "properties.line_items.items.properties.user_id", - "properties.line_items.items.properties.compare_at_price", - "properties.gateway", - "properties.shipping_lines.items.properties.delivery_category", - "properties.shipping_lines.items.properties.validation_context", - "properties.shipping_lines.items.properties.requested_fulfillment_service_id", - "properties.shipping_lines.items.properties.tax_lines.items.properties.identifier", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_name", - "properties.shipping_lines.items.properties.tax_lines.items.properties.tax_api_client_id", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_exempt_amount", - "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_source", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_code", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_jurisdiction_type", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_taxable_amount", - "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_type", - "properties.shipping_lines.items.properties.tax_lines.items.properties.tax_type", - "properties.shipping_lines.items.properties.tax_lines.items.properties.jurisdiction_id", - "properties.shipping_lines.items.properties.tax_lines.items.properties.reporting_non_taxable_amount", - "properties.shipping_lines.items.properties.custom_tax_lines", - "properties.shipping_lines.items.properties.estimated_delivery_time_range", - "properties.line_items.items.properties.tax_lines.items.properties.tax_type", - "properties.line_items.items.properties.tax_lines.items.properties.identifier", - "properties.line_items.items.properties.discount_allocations.items.properties.id", - "properties.line_items.items.properties.discount_allocations.items.properties.created_at", - ], - ], - [ - /collection_listings\/.+/, - ["properties.collection_listing.properties.default_product_image", "properties.collection_listing.properties.image"], - ], - [/domains\/.+/, ["properties.localization.properties.country"]], - [/orders\/.+/, ["properties.client_details.properties.session_hash"]], -]; - -// example data we feed the schema infer-er for each topic to allow it to discover real types -const manualExamples: [RegExp, Record][] = [ - [ - /.+/, - { - admin_graphql_api_id: "gid://shopify/Something/1234567890", - admin_graphql_api_job_id: "gid://shopify/Job/1234567890", - created_at: "2021-12-30T19:00:00-05:00", - updated_at: "2021-12-30T19:00:00-05:00", - address2: "Apt 123", - latitude: 10.1, - longitude: 10.1, - location_id: 111111, - }, - ], - [ - /(app|shop)\/.+/, - { - domain: "example.com", - source: "example source", - myshopify_domain: "example.myshopify.com", - google_apps_domain: "example.com", - google_apps_login_enabled: true, - password_enabled: true, - taxes_included: true, - tax_shipping: true, - iana_timezone: "America/New_York", - auto_configure_tax_inclusivity: true, - county_taxes: true, - }, - ], - [ - /bulk_operations\/.+/, - { - error_code: "SOME_ERROR_ENUM", - }, - ], - [ - /carts\/.+/, - { - note: "some cart note string", - }, - ], - [ - /(checkouts|orders)\/.+/, - { - gateway: "shopify_payments", - landing_site: "https://example.com", - note: "some order note", - referring_site: "https://example.com", - completed_at: "2021-12-30T19:00:00-05:00", - closed_at: "2021-12-30T19:00:00-05:00", - user_id: 11111111, - location_id: 22222222, - source_identifier: "some_source_identifier", - source_url: "https://example.com", - device_id: "some_device_id", - phone: "+1 (123) 456 7890", - sms_marketing_phone: "+1 (123) 456 7890", - customer_locale: "en", - source: "some_source", - total_duties: 10.11, - app_id: 12345, - browser_ip: "10.0.0.1", - cart_token: "some_cart_token", - checkout_id: 12345, - client_details: { - accept_language: "en-US,en;q=0.9", - browser_height: 800, - }, - confirmation_number: "some_confirmation_number", - current_total_additional_fees_set: { - shop_money: { - amount: "0.00", - currency_code: "USD", - }, - presentment_money: { - amount: "0.00", - currency_code: "USD", - }, - }, - current_total_duties_set: { - shop_money: { - amount: "0.00", - currency_code: "USD", - }, - presentment_money: { - amount: "0.00", - currency_code: "USD", - }, - }, - original_total_additional_fees_set: { - shop_money: { - amount: "0.00", - currency_code: "USD", - }, - presentment_money: { - amount: "0.00", - currency_code: "USD", - }, - }, - original_total_duties_set: { - shop_money: { - amount: "0.00", - currency_code: "USD", - }, - presentment_money: { - amount: "0.00", - currency_code: "USD", - }, - }, - checkout_token: "some_checkout_token", - landing_site_ref: "https://example.com", - merchant_of_record_app_id: 12345, - po_number: "some_po_number", - processed_at: "2021-12-30T19:00:00-05:00", - reference: "some_reference", - payment_terms: "some_payment_terms", - reservation_token: "some_reservation_token", - billing_address: { - address2: "suite 101", - latitude: 34.1, - longitude: 34.1, - }, - shipping_address: { - address2: "suite 101", - latitude: 34.1, - longitude: 34.1, - }, - customer: { - created_at: "2024-05-05T02:42:32+08:00", - marketing_opt_in_level: "single_opt_in", - note: "some string", - multipass_identifier: "hello", - }, - }, - ], - [ - /checkouts\/.+/, - { - line_items: [ - { - presentment_variant_title: "51 x 76 / Blanc / 1", - taxable: true, - variant_price: "69.00", - presentment_title: "Taie d'oreiller Pure Soie de Mûrier", - requires_shipping: true, - unit_price_measurement: { - measured_type: null, - quantity_unit: null, - quantity_value: null, - reference_unit: null, - reference_value: null, - }, - variant_title: "51 x 76 / Blanc / 1", - title: "Taie d'oreiller Pure Soie de Mûrier", - gift_card: false, - destination_location_id: 4042942054746, - compare_at_price: null, - key: "e0105d4efb1970501cf831566fa79752", - line_price: "69.00", - vendor: "Emily's pillow", - quantity: 1, - applied_discounts: [], - grams: 110, - properties: [ - { - name: "_isJust", - value: "true", - }, - ], - tax_lines: [ - { - reporting_jurisdiction_type: null, - reporting_non_taxable_amount: "0.00", - zone: null, - compare_at: 0.2, - reporting_taxable_amount: "57.50", - tax_api_client_id: null, - jurisdiction_source: "ActiveTax", - title: "FR TVA", - identifier: null, - jurisdiction_type: "COUNTRY", - reporting_jurisdiction_code: null, - source: "MerchantActiveTax", - reporting_exempt_amount: "0.00", - reporting_jurisdiction_name: null, - jurisdiction_id: "FR", - tax_type: null, - channel_liable: false, - price: "11.50", - rate: 0.2, - position: 1, - tax_calculation_price: "11.50", - }, - ], - sku: "EMIL-TO5176-22WH", - rank: 0, - user_id: null, - product_id: 4610455470216, - discount_allocations: [], - origin_location_id: 1743669821576, - fulfillment_service: "manual", - variant_id: 32516187029640, - price: "69.00", - }, - ], - }, - ], - [ - /collections\/.+/, - { - sort_order: "manual", - template_suffix: "some_template_suffix", - }, - ], - [ - /collection_listings\/.+/, - { - collection_listing: { - updated_at: "2021-12-30T19:00:00-05:00", - sort_order: 1, - }, - }, - ], - [ - /company_locations\/.+/, - { - buyer_experience_configuration: { pay_now_only: true }, - billing_address: { - address2: "suite 101", - }, - shipping_address: { - address2: "suite 101", - }, - }, - ], - [ - /customers\/.+/, - { - last_order_id: 12345, - multipass_identifier: "some_multipass_identifier", - last_order_name: "Foobar", - phone: "+1 (123) 456 7890", - sms_marketing_consent: false, - email_marketing_consent: false, - accepts_marketing_updated_at: "2021-12-30T19:00:00-05:00", - marketing_opt_in_level: "single_opt_in", - }, - ], - [ - /customer_account_settings\/.+/, - { - url: "https://example.com", - }, - ], - [ - /customer.+consent\/.+/, - { - phone: "+1 (123) 456 7890", - email_address: "test@test.com", - }, - ], - [ - /disputes\/.+/, - { - evidence_sent_on: "2021-12-30T19:00:00-05:00", - finalized_on: "2021-12-30T19:00:00-05:00", - }, - ], - [ - /draft_orders\/.+/, - { - invoice_sent_at: "2021-12-30T19:00:00-05:00", - order_id: 12345, - }, - ], - [ - /fulfillment_events\/.+/, - { - city: "Ottawa", - province: "ON", - zip: "K1P1J1", - address1: "150 Elgin St", - estimated_delivery_at: "2021-12-30T19:00:00-05:00", - }, - ], - [ - /fulfillment_orders\/.+/, - { - remaining_fulfillment_order: { - id: 5859333242902, - shop_id: 20978040854, - order_id: 4804938989590, - assigned_location_id: 67794436118, - request_status: "unsubmitted", - status: "open", - supported_actions: ["request_fulfillment", "hold", "move"], - destination: { - id: 5479404371990, - address1: "23 Hassall Street", - address2: "", - city: "Parramatta", - company: null, - country: "Australia", - email: "", - first_name: "Tyler", - last_name: "Kelleher", - phone: null, - province: "New South Wales", - zip: "2150", - }, - line_items: [ - { - id: 12675478814742, - shop_id: 20978040854, - fulfillment_order_id: 5859333242902, - quantity: 1, - line_item_id: 12553770336278, - inventory_item_id: 44276125368342, - fulfillable_quantity: 1, - variant_id: 42182036422678, - }, - ], - fulfill_at: "2022-10-13T13:00:00-04:00", - fulfill_by: null, - international_duties: { - incoterm: "DAP", - }, - fulfillment_holds: [], - delivery_method: { - id: 140816351254, - method_type: "shipping", - min_delivery_date_time: null, - max_delivery_date_time: null, - }, - assigned_location: { - address1: null, - address2: null, - city: null, - country_code: "CA", - location_id: 67794436118, - name: "test-created-via-api-2", - phone: null, - province: null, - zip: null, - }, - }, - }, - ], - [ - /fulfillments\/.+/, - { - service: "manual", - shipment_status: "confirmed", - - origin_address: { - first_name: "Steve", - address1: "123 Shipping Street", - phone: "555-555-SHIP", - city: "Shippington", - zip: "40003", - province: "Kentucky", - country: "United States", - last_name: "Shipper", - address2: null, - company: "Shipping Company", - latitude: null, - longitude: null, - name: "Steve Shipper", - country_code: "US", - province_code: "KY", - }, - }, - ], - [ - /inventory_items\/.+/, - { - cost: 10.11, - country_code_of_origin: "CA", - province_code_of_origin: "CA", - harmonized_system_code: "1234567890", - }, - ], - [ - /inventory_levels\/.+/, - { - available: 10, - }, - ], - [ - /order_transactions\/.+/, - { - message: "some message from the gateway", - user_id: 12345, - parent_id: 12345, - processed_at: "2021-12-30T19:00:00-05:00", - device_id: "some_device_id", - error_code: "SOME_ERROR_ENUM", - }, - ], - [ - /orders\/risk_assessment.+/, - { - provider_id: 12345, - provider_title: "whatever", - }, - ], - [ - /products\/.+/, - { - template_suffix: "something", - image: "gid://shopify/ProductImage/1234567890", - }, - ], - [ - /refunds\/.+/, - { - return: "unknown", - }, - ], - [ - /selling_plan_groups\/.+/, - { - app_id: 12345, - description: "some description", - position: 1, - }, - ], - [ - /subscription_billing_attempts\/.+/, - { - id: 12345, - error_message: "some error message", - error_code: "some error copde", - }, - ], - [ - /subscription_billing_cycle.+/, - { - contract_edit: "some contract edit", - }, - ], - [ - /tender_transactions\/.+/, - { - user_id: 12345, - processed_at: "2021-12-30T19:00:00-05:00", - payment_details: { something: "true" }, - }, - ], -]; - -const shippingAddress = { - type: "object", - properties: { - first_name: { - type: "string", - }, - address1: { - type: "string", - }, - phone: { - type: "string", - }, - city: { - type: "string", - }, - zip: { - type: "string", - }, - province: { - type: "string", - }, - country: { - type: "string", - }, - last_name: { - type: "string", - }, - address2: { - type: ["string", "null"], - }, - company: { - type: ["string", "null"], - }, - latitude: { - type: ["number", "null"], - }, - longitude: { - type: ["number", "null"], - }, - name: { - type: "string", - }, - country_code: { - type: "string", - }, - province_code: { - type: "string", - }, - }, -}; - -const overrides: { topics: string[]; schema: any; versions?: string[] }[] = [ - { - topics: ["checkouts/create", "checkouts/update"], - schema: { - shipping_address: shippingAddress, - }, - }, - { - topics: ["order_transactions/create", "order_transactions/update"], - schema: { - receipt: { - type: "object", - additionalProperties: true, - }, - }, - }, -]; From 87e56820422a6aaf587e9112e8ea3478c63dc972 Mon Sep 17 00:00:00 2001 From: John Duff Date: Wed, 30 Apr 2025 10:45:19 -0400 Subject: [PATCH 2/3] Export inferSchemaFromExamplePayload and add tests --- src/index.ts | 3 + test/index.test.ts | 161 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 163 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index d775653..43b47da 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,9 @@ import fs from "fs"; import path from "path"; import fastGlob from "fast-glob"; +import { inferSchemaFromExamplePayload } from "./infer-schema"; + +export { inferSchemaFromExamplePayload }; const loaded: { [kind in "metadatas" | "schemas"]: { [apiVersion: string]: { [topic: string]: any } } } = { metadatas: {}, schemas: {} }; diff --git a/test/index.test.ts b/test/index.test.ts index c8c58fd..cefad3f 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert"; -import { allTopicsForVersion, metadataForWebhook, schemaForWebhook } from "../src/index"; +import { allTopicsForVersion, metadataForWebhook, schemaForWebhook, inferSchemaFromExamplePayload } from "../src/index"; void test("can get the schema for an example topic", () => { assert.ok(schemaForWebhook("2024-04", "products/create")); @@ -24,3 +24,162 @@ void test("throws an error if passed an invalid version", () => { } assert.ok(threw); }); + +const EXPECTED_SCHEMA = { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + id: { type: "integer" }, + options: { + type: "object", + properties: { + name: { type: "string" }, + value: { type: "string" }, + }, + required: ["name", "value"], + }, + variants: { + type: "array", + items: { + type: "object", + properties: { + id: { type: "integer" }, + title: { type: "string" }, + }, + required: ["id", "title"], + }, + }, + }, + required: ["id", "options", "variants"], + type: "object", +}; + +void test("can infer the schema from an example payload", () => { + const result = inferSchemaFromExamplePayload( + { id: 1, variants: [{ id: 2, title: "test" }], options: { name: "color", value: "red" } }, + { name: "test" } + ); + + assert.deepEqual(result.schema, EXPECTED_SCHEMA); + assert.equal(result.warnings, 0); + assert.equal(result.errors.length, 0); +}); + +void test("can infer the schema from an example payload with a null value", () => { + const result = inferSchemaFromExamplePayload({ id: null, variants: [] }, { name: "test" }); + + assert.deepEqual(result.schema, { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + id: { + type: "null", + }, + variants: { + items: false, + type: "array", + }, + }, + required: ["id", "variants"], + type: "object", + }); + assert.equal(result.warnings, 1); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].message, "null type found in final schema"); + assert.equal(result.errors[0].path, "properties.id"); +}); + +void test("infer schema uses payload overrides", () => { + const result = inferSchemaFromExamplePayload({ created_at: null, billing_address: null }, { name: "orders/updated" }); + + assert.deepEqual(result.schema, { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + billing_address: { + properties: { + address2: { + type: "string", + }, + latitude: { + type: "number", + }, + longitude: { + type: "number", + }, + }, + required: ["address2", "latitude", "longitude"], + type: ["null", "object"], + }, + created_at: { + format: "date-time", + type: ["null", "string"], + }, + }, + required: ["billing_address", "created_at"], + type: "object", + }); + assert.equal(result.warnings, 0); + assert.equal(result.errors.length, 0); +}); + +void test("infer schema does not apply payload overrides if the topic doesn't match", () => { + const result = inferSchemaFromExamplePayload({ created_at: null, billing_address: null }, { name: "products/updated" }); + + assert.deepEqual(result.schema, { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + billing_address: { + type: "null", + }, + created_at: { + format: "date-time", + type: ["null", "string"], + }, + }, + required: ["billing_address", "created_at"], + type: "object", + }); + assert.equal(result.warnings, 1); + assert.equal(result.errors.length, 1); + assert.equal(result.errors[0].message, "null type found in final schema"); + assert.equal(result.errors[0].path, "properties.billing_address"); +}); + +void test("infer schema uses schema overrides", () => { + const result = inferSchemaFromExamplePayload({ receipt: { id: 1 } }, { name: "order_transactions/update" }); + + assert.deepEqual(result.schema, { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + receipt: { + type: "object", + additionalProperties: true, + }, + }, + required: ["receipt"], + type: "object", + }); + assert.equal(result.warnings, 0); + assert.equal(result.errors.length, 0); +}); + +void test("infer schema does not apply schema overrides if the topic doesn't match", () => { + const result = inferSchemaFromExamplePayload({ receipt: { id: 1 } }, { name: "products/updated" }); + + assert.deepEqual(result.schema, { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: { + receipt: { + type: "object", + properties: { + id: { + type: "integer", + }, + }, + required: ["id"], + }, + }, + required: ["receipt"], + type: "object", + }); + assert.equal(result.warnings, 0); + assert.equal(result.errors.length, 0); +}); From c620dea8d2161a7d5cc99b3f8956da090dbccf20 Mon Sep 17 00:00:00 2001 From: John Duff Date: Wed, 30 Apr 2025 10:47:03 -0400 Subject: [PATCH 3/3] Move packages needed for schema inference to dependencies --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 22581a8..9c9c38d 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,9 @@ "license": "MIT", "devDependencies": { "@arethetypeswrong/cli": "^0.15.3", - "@cloudflare/json-schema-walker": "^0.1.1", "@gadgetinc/eslint-config": "^0.6.1", "@gadgetinc/prettier-config": "^0.4.0", "@jsonhero/json-infer-types": "^1.2.11", - "@jsonhero/schema-infer": "^0.1.5", "@opentelemetry/api": "^1.8.0", "@types/fs-extra": "^11.0.4", "@types/lodash": "^4.17.0", @@ -48,16 +46,18 @@ "globby": "^14.0.1", "got": "^11.8.6", "inflected": "^2.1.0", - "lodash": "^4.17.21", "prettier": "^2.8.8", "publint": "^0.2.7", - "safe-stable-stringify": "^2.5.0", "traverse": "^0.6.9", "tsx": "^4.7.3", "typescript": "^5.4.5" }, "dependencies": { - "fast-glob": "^3.3.2" + "fast-glob": "^3.3.2", + "@cloudflare/json-schema-walker": "^0.1.1", + "@jsonhero/schema-infer": "^0.1.5", + "lodash": "^4.17.21", + "safe-stable-stringify": "^2.5.0" }, "packageManager": "pnpm@9.11.0+sha512.0a203ffaed5a3f63242cd064c8fb5892366c103e328079318f78062f24ea8c9d50bc6a47aa3567cabefd824d170e78fa2745ed1f16b132e16436146b7688f19b" }