diff --git a/README.md b/README.md index 963a3ac..352e459 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,15 @@ firebase ext:install typesense/firestore-typesense-search --project=[your-projec When you install this extension, you'll be able to configure the following parameters: -| Parameter | Description | -|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Firestore Collection Path | The Firestore collection that needs to be indexed into Typesense. | -| Firestore Collection Fields | A comma separated list of fields that need to be indexed from each Firestore document. Leave blank to index all fields. | -| Flatten Nested Documents | Should nested documents in Firestore be flattened before they are indexed in Typesense? Set to "Yes" for Typesense Server versions v0.23.1 and below, since indexing Nested objects is natively supported only in Typesense Server v0.24 and above. | -| Typesense Hosts | A comma-separated list of Typesense Hosts (only domain without https or port number). For single node clusters, a single hostname is sufficient. For multi-node Highly Available or (Search Delivery Network) SDN Clusters, please be sure to mention all hostnames in a comma-separated list. | -| Typesense API Key | A Typesense API key with admin permissions. Click on "Generate API Key" in cluster dashboard in Typesense Cloud. | -| Typesense Collection Name | Typesense collection name to index data into (you need to create this collection in Typesense yourself. This extension does not create the Typesense Collection for you). | -| Cloud Functions location | Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). | +| Parameter | Description | +|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Firestore Collection Path | The Firestore collection that needs to be indexed into Typesense. | +| Firestore Collection Fields | A comma separated list of fields that need to be indexed from each Firestore document. Leave blank to index all fields. With a "=" separated another field name can be specified for a field, e.g. "field1=otherFieldName1". Thus the firestore field "field1" in typesense is called "otherFieldName1". | +| Flatten Nested Documents | Should nested documents in Firestore be flattened before they are indexed in Typesense? Set to "Yes" for Typesense Server versions v0.23.1 and below, since indexing Nested objects is natively supported only in Typesense Server v0.24 and above. | +| Typesense Hosts | A comma-separated list of Typesense Hosts (only domain without https or port number). For single node clusters, a single hostname is sufficient. For multi-node Highly Available or (Search Delivery Network) SDN Clusters, please be sure to mention all hostnames in a comma-separated list. | +| Typesense API Key | A Typesense API key with admin permissions. Click on "Generate API Key" in cluster dashboard in Typesense Cloud. | +| Typesense Collection Name | Typesense collection name to index data into (you need to create this collection in Typesense yourself. This extension does not create the Typesense Collection for you). | +| Cloud Functions location | Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). | > ⚠️ You'll notice that there is no way to configure the port number or protocol. This is because this extension only supports connecting to Typesense running HTTPS on Port 443, since your data goes from Firebase to Typesense over the public internet and we want your data to be encrypted in transit. diff --git a/extension.yaml b/extension.yaml index d2007ba..14a0ab5 100644 --- a/extension.yaml +++ b/extension.yaml @@ -69,8 +69,8 @@ params: - param: FIRESTORE_COLLECTION_FIELDS label: Firestore Collection Fields description: >- - A comma separated list of fields that need to be indexed from each Firestore document. Leave blank to index all fields. - example: field1,field2,field3 + A comma separated list of fields that need to be indexed from each Firestore document. Leave blank to index all fields. With a "=" separated another field name can be specified for a field, e.g. "field1=otherFieldName1". Thus the firestore field "field1" in typesense is called "otherFieldName1". + example: field1,field2=otherFieldName2,field3 default: "" required: false - param: FLATTEN_NESTED_DOCUMENTS diff --git a/functions/src/backfillToTypesenseFromFirestore.js b/functions/src/backfillToTypesenseFromFirestore.js index 01879ae..696f7f3 100644 --- a/functions/src/backfillToTypesenseFromFirestore.js +++ b/functions/src/backfillToTypesenseFromFirestore.js @@ -31,7 +31,7 @@ const validateBackfillRun = (snapshot) => { module.exports = functions.handler.firestore.document .onWrite(async (snapshot, context) => { functions.logger.info("Backfilling " + - `${config.firestoreCollectionFields.join(",")} fields in Firestore documents ` + + `${Array.from(config.firestoreCollectionFields.keys()).join(",")} fields in Firestore documents ` + `from ${config.firestoreCollectionPath} ` + `into Typesense Collection ${config.typesenseCollectionName} ` + `on ${config.typesenseHosts.join(",")}`); diff --git a/functions/src/config.js b/functions/src/config.js index ed6c78e..2f2941a 100644 --- a/functions/src/config.js +++ b/functions/src/config.js @@ -1,10 +1,27 @@ +/** + * @param {string} fieldNames + * @return {Map} map of field names where key is the field name in Firestore and value is the field name in Typesense + * @example + * parseFieldNames("foo=bar, baz") => Map { "foo" => "bar", "baz" => "baz" } + * parseFieldNames("foo=bar, baz=qux") => Map { "foo" => "bar", "baz" => "qux" } + * parseFieldNames("foo=bar, baz=qux,") => Map { "foo" => "bar", "baz" => "qux" } + * parseFieldNames("foo, baz = qux, bar , ") => Map { "foo" => "foo", "baz" => "qux", "bar" => "bar" } + */ +const parseFieldNames = (fieldNames) => new Map( + fieldNames.split(",") + .filter((v) => v) + .map( + (f) => { + const [key, value = key] = f.split("=").map((p) => p.trim()); + return [key, value]; + }, + ), +); + module.exports = { firestoreCollectionPath: process.env.FIRESTORE_COLLECTION_PATH, firestoreCollectionFields: - (process.env.FIRESTORE_COLLECTION_FIELDS || "") - .split(",") - .map((f) => f.trim()) - .filter((f) => f), + parseFieldNames(process.env.FIRESTORE_COLLECTION_FIELDS || ""), shouldFlattenNestedDocuments: process.env.FLATTEN_NESTED_DOCUMENTS === "true", typesenseHosts: process.env.TYPESENSE_HOSTS.split(",").map((e) => e.trim()), @@ -15,3 +32,4 @@ module.exports = { typesenseBackfillTriggerDocumentInFirestore: "typesense_sync/backfill", typesenseBackfillBatchSize: 1000, }; + diff --git a/functions/src/utils.js b/functions/src/utils.js index 602a8d0..7856921 100644 --- a/functions/src/utils.js +++ b/functions/src/utils.js @@ -22,7 +22,7 @@ const mapValue = (value) => { /** * @param {DocumentSnapshot} firestoreDocumentSnapshot - * @param {Array} fieldsToExtract + * @param {Map} fieldsToExtract * @return {Object} typesenseDocument */ exports.typesenseDocumentFromSnapshot = ( @@ -33,12 +33,12 @@ exports.typesenseDocumentFromSnapshot = ( let entries = Object.entries(data); - if (fieldsToExtract.length) { - entries = entries.filter(([key]) => fieldsToExtract.includes(key)); + if (fieldsToExtract.size) { + entries = entries.filter(([key]) => fieldsToExtract.has(key)); } // Build a document with just the fields requested by the user, and mapped from Firestore types to Typesense types - const mappedDocument = Object.fromEntries(entries.map(([key, value]) => [key, mapValue(value)])); + const mappedDocument = Object.fromEntries(entries.map(([key, value]) => [fieldsToExtract.get(key), mapValue(value)])); // using flat to flatten nested objects for older versions of Typesense that did not support nested fields // https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields