ZAP: Multi-protocol Configurator Integration (#1276)

> - Schema changes such that sessions now have session partitions to allow multiple zcl and template config files.
> - Session packages and endpoint types are now linked to a session partition such that generation and UI can be made exclusive to the device type selected on an endpoint
> - All relevant queries have been updated with session partition
> - imp-exp of .zap files is taken into account here as well
> - ZAP Generation can now generate the corresponding templates based on the category of the device type and the category of the generation templates eg: matter or zigbee. The same goes for UI
> - All existing tests pass with these changes
- Fixing the cluster view when multiple zcl device types are selected
- ZAP: Opening existing .zap files does not allow multi-device type selection
- Fixing sessionAttempt in initialize.js and cleaning up the packages from the session in jsonDataLoader function in import-json fixed this
- Other minor cleanup
- Fixing CMP generation for command line in startup.js#generateSingleFile such that it generates for all generation template json files and not just one of them
- bumping the feature level of zap
- Adding the reference to parentEndpointRef in endpoint extended in db-mapping due to rebase and also fixing the selectAllEndpointsBasedOnTemplateCategory for the same
- Cleaning up session package and session partitions to avoid having the wrong session packages and partitions in deleteAllSessionPackages
- Cleaning up query-session#ensureBlankSession
- Fixing the zap schema for command trigger warnings when adding endpoints
- Fixing the parent-endpoint.test.js just like the rest with session partition info
- Fixing the default attribute enablement based on catgeory such as matter/zigbee for global attributes
other minor cleanup
- Not showing the extensions tab in case of multi-protocol since it is currently not supported, see ZCLToolbar.vue
- Showing all logos based on the categories from the zcl files selected, see ZCLToolbar.vue
- Added multi-protocol tests, see multi-protocol.test.js, multi-protocol.zap, test-utils.js
- Removed lo-devices.xml from the zcl-with-extensions.json matter file to test multi-protocol more efficiently since for generation as is the case in reality as well.
- Fixing calls to insertEndpointType with device type params in query.test.js
- Fix cypress tests for vue tour which was happening due to category(zigbee/matter) not being picked from zcl properties files. Also handling the default use case when category is not found
- Calling this..dispatch('zap/updateClusters') in vue tour after update to endpoint type due to multi-protocol changes
- Fixing the default use case for showing and not showing UI elements in ui-options.js
- Fixing server-bare.test.js when inserting a package along with some other minor cleanup which is needed
- Fix the server-bare.test.js
- Improving the session creation and package selection process in the UI via ZapConfig.vue changes
- Pre-selecting the package files which exist in ZapConfig.vue. Also now showing package selection view if the packages which exist in the db are the same as the ones in the .zap file.
- Showing a warning in the custom xml page when a multi-proptocol config is open since custom xml is not currently supported for custom xml. See ZclCustomZclView.vue
- JIRA: ZAPP-1216

Author: brdandu

apack.json

@@ -4,7 +4,7 @@
   "description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
   "path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
   "requiredFeatureLevel": "apack.core:9",
-  "featureLevel": 100,
+  "featureLevel": 101,
   "uc.triggerExtension": "zap",
   "executable": {
     "zap:win32.x86_64": {

docs/zap-schema.svg

@@ -58,6 +58,7 @@
     "package-metadata": "node src-script/zap-package-metadata.js",
     "zap": "node src-script/zap-start.js --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json",
     "zapall": "node src-script/zap-start.js --logToStdout --zcl ./test/resource/meta/zcl.json --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json --gen ./test/resource/meta/gen-test.json",
+    "zapmultiprotocol": "node src-script/zap-start.js --logToStdout --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl-with-test-extensions.json --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json",
     "zapzigbee": "node src-script/zap-start.js --logToStdout --zcl ./zcl-builtin/silabs/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json",
     "zapmatter": "node src-script/zap-start.js --logToStdoutc --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/matter/gen-test.json",
     "zapmeta": "node src-script/zap-start.js --logToStdout --zcl ./test/resource/meta/zcl.json --gen ./test/resource/meta/gen-test.json --in ./test/resource/test-meta.zap",
@@ -66,6 +67,7 @@
     "zigbeezap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/zigbee-server/ --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --reuseZapInstance",
     "matterzap-devserver": "node src-script/zap-start.js server --stateDirectory ~/.zap/matter-server/ --allowCors --logToStdout --zcl ./zcl-builtin/matter/zcl.json --gen ./test/resource/meta/gen-test.json --reuseZapInstance",
     "matterzap-devserver2": "node src-script/zap-start.js server --stateDirectory --zcl ./zcl-builtin/matter/zcl.json --allowCors --logToStdout --gen ./test/gen-template/matter3/t.json --reuseZapInstance",
+    "zapall-devserver": "node src-script/zap-start.js server --stateDirectory --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --allowCors --logToStdout --gen ./test/gen-template/zigbee/gen-templates.json --gen ./test/gen-template/matter/gen-test.json --gen --reuseZapInstance",
     "server": "node src-script/zap-start.js server --stateDirectory ~/.zap/server/ --allowCors --logToStdout --zcl ./zcl-builtin/silabs/zcl.json  --zcl ./zcl-builtin/matter/zcl.json --gen ./test/gen-template/zigbee/gen-templates.json  --gen ./test/gen-template/matter/gen-test.json ",
     "stop": "node src-script/zap-start.js stop --reuseZapInstance",
     "status": "node src-script/zap-start.js status --reuseZapInstance",

package.json

@@ -384,6 +384,23 @@ exports.map = {
       name: x.NAME,
       caption: x.DESCRIPTION,
       class: x.CLASS,
+      packageRef: x.PACKAGE_REF,
+    }
+  },
+
+  deviceTypeExtended: (x) => {
+    if (x == null) return undefined
+    return {
+      id: x.DEVICE_TYPE_ID,
+      code: x.CODE,
+      profileId: x.PROFILE_ID,
+      domain: x.DOMAIN,
+      label: x.NAME,
+      name: x.NAME,
+      caption: x.DESCRIPTION,
+      class: x.CLASS,
+      packageRef: x.PACKAGE_REF,
+      category: x.CATEGORY,
     }
   },
 
@@ -452,6 +469,25 @@ exports.map = {
       parentEndpointIdentifier: x.PARENT_ENDPOINT_IDENTIFIER,
     }
   },
+  endpointExtended: (x) => {
+    if (x == null) return undefined
+    return {
+      id: x.ENDPOINT_ID,
+      endpointRef: x.ENDPOINT_ID,
+      sessionRef: x.SESSION_REF,
+      endpointIdentifier: x.ENDPOINT_IDENTIFIER,
+      endpointId: x.ENDPOINT_IDENTIFIER,
+      endpointTypeRef: x.ENDPOINT_TYPE_REF,
+      profileId: x.PROFILE,
+      networkId: x.NETWORK_IDENTIFIER,
+      endpointVersion: x.DEVICE_VERSION, // Left for backwards compatibility
+      deviceVersion: x.DEVICE_VERSION,
+      deviceIdentifier: x.DEVICE_IDENTIFIER,
+      parentRef: x.PARENT_ENDPOINT_REF,
+      parentEndpointIdentifier: x.PARENT_ENDPOINT_IDENTIFIER,
+      category: x.CATEGORY, // Category of the device type coming from the zcl package it belongs to.
+    }
+  },
   endpointType: (x) => {
     if (x == null) return undefined
     return {
@@ -643,6 +679,8 @@ exports.map = {
       sessionRef: x.SESSION_REF,
       required: x.REQUIRED,
       type: x.TYPE,
+      sessionPartitionId: x.SESSION_PARTITION_ID,
+      category: x.CATEGORY,
     }
   },
   sessionLog: (x) => {
@@ -664,6 +702,16 @@ exports.map = {
       newNotification: x.NEW_NOTIFICATION == 1,
     }
   },
+
+  sessionPartition: (x) => {
+    if (x == null) return undefined
+    return {
+      sessionPartitionId: x.SESSION_PARTITION_ID,
+      sessionRef: x.SESSION_REF,
+      sessionPartitionNumber: x.SESSION_PARTITION_NUMBER,
+    }
+  },
+
   user: (x) => {
     if (x == null) return undefined
     return {

src-electron/db/db-mapping.js

@@ -1103,8 +1103,12 @@ INNER JOIN
   CLUSTER
 ON
   ENDPOINT_TYPE_CLUSTER.CLUSTER_REF = CLUSTER.CLUSTER_ID
+INNER JOIN
+  SESSION_PARTITION
+ON
+ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
 WHERE
-  ENDPOINT_TYPE.SESSION_REF = ${sessionId}
+  SESSION_PARTITION.SESSION_REF = ${sessionId}
 AND
   ENDPOINT_TYPE_CLUSTER.ENABLED=1
 AND

src-electron/db/query-attribute.js

@@ -218,8 +218,12 @@ async function selectAllUserClustersWithTokenAttributes(
     CLUSTER
   ON
     ENDPOINT_TYPE_CLUSTER.CLUSTER_REF = CLUSTER.CLUSTER_ID
+  INNER JOIN
+    SESSION_PARTITION
+  ON
+  ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
   WHERE
-    ENDPOINT_TYPE.SESSION_REF = ?
+    SESSION_PARTITION.SESSION_REF = ?
   AND
     ENDPOINT_TYPE_CLUSTER.ENABLED=1
   AND

src-electron/db/query-cluster.js

@@ -27,6 +27,7 @@ const dbEnum = require('../../src-shared/db-enum.js')
 const queryZcl = require('./query-zcl.js')
 const queryUpgrade = require('../matter/matter.js')
 const queryDeviceType = require('./query-device-type')
+const querySession = require('./query-session')
 const queryCommand = require('./query-command.js')
 const restApi = require('../../src-shared/rest-api.js')
 const _ = require('lodash')
@@ -575,16 +576,18 @@ async function selectCountOfEndpointsWithGivenEndpointIdentifier(
 /**
  * Promises to add an endpoint type.
  *
- * @export
  * @param {*} db
- * @param {*} sessionId
+ * @param {*} sessionPartitionInfo
  * @param {*} name
  * @param {*} deviceTypeRef
+ * @param {*} deviceTypeIdentifier
+ * @param {*} deviceTypeVersion
+ * @param {*} doTransaction
  * @returns Promise to update endpoints.
  */
 async function insertEndpointType(
   db,
-  sessionId,
+  sessionPartitionInfo,
   name,
   deviceTypeRef,
   deviceTypeIdentifier,
@@ -603,8 +606,8 @@ async function insertEndpointType(
   // Insert endpoint type
   let newEndpointTypeId = await dbApi.dbInsert(
     db,
-    'INSERT OR REPLACE INTO ENDPOINT_TYPE ( SESSION_REF, NAME ) VALUES ( ?, ?)',
-    [sessionId, name]
+    'INSERT OR REPLACE INTO ENDPOINT_TYPE ( SESSION_PARTITION_REF, NAME ) VALUES ( ?, ?)',
+    [sessionPartitionInfo.sessionPartitionId, name]
   )
 
   // Creating endpoint type and device type ref combinations along with order of insertion
@@ -634,7 +637,7 @@ async function insertEndpointType(
       db,
       'ERROR',
       isErrorStringPresent ? err.split('Error:')[1] : err,
-      sessionId,
+      sessionPartitionInfo.sessionRef,
       1,
       1
     )
@@ -655,7 +658,7 @@ async function insertEndpointType(
   for (const dtRef of deviceTypeRefs) {
     await setEndpointDefaults(
       db,
-      sessionId,
+      sessionPartitionInfo.sessionRef,
       newEndpointTypeId,
       dtRef,
       doTransaction
@@ -678,7 +681,8 @@ async function duplicateEndpointType(db, endpointTypeId) {
     db,
     `
     SELECT
-      ENDPOINT_TYPE.SESSION_REF,
+      SESSION_PARTITION.SESSION_REF,
+      SESSION_PARTITION.SESSION_PARTITION_ID,
       ENDPOINT_TYPE.NAME,
       ENDPOINT_TYPE_DEVICE.DEVICE_TYPE_REF,
       ENDPOINT_TYPE_DEVICE.DEVICE_IDENTIFIER,
@@ -689,6 +693,10 @@ async function duplicateEndpointType(db, endpointTypeId) {
       ENDPOINT_TYPE_DEVICE
     ON
       ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
     WHERE
       ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF = ?`,
     [endpointTypeId]
@@ -699,9 +707,12 @@ async function duplicateEndpointType(db, endpointTypeId) {
     // Enter into the endpoint_type table
     newEndpointTypeId = await dbApi.dbInsert(
       db,
-      `INSERT INTO ENDPOINT_TYPE (SESSION_REF, NAME)
+      `INSERT INTO ENDPOINT_TYPE (SESSION_PARTITION_REF, NAME)
       VALUES (?, ?)`,
-      [endpointTypeDeviceInfo[0].SESSION_REF, endpointTypeDeviceInfo[0].NAME]
+      [
+        endpointTypeDeviceInfo[0].SESSION_PARTITION_REF,
+        endpointTypeDeviceInfo[0].NAME,
+      ]
     )
 
     // Enter into the endpoint_type_device table to establish the endpoint_type
@@ -758,8 +769,12 @@ async function updateEndpointType(db, sessionId, endpointTypeId, changesArray) {
     INNER JOIN
       ENDPOINT_TYPE_DEVICE
     ON ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ENDPOINT_TYPE_DEVICE.ENDPOINT_TYPE_REF
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      ENDPOINT_TYPE.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
     WHERE
-      ENDPOINT_TYPE_ID = ? AND SESSION_REF = ?
+      ENDPOINT_TYPE.ENDPOINT_TYPE_ID = ? AND SESSION_PARTITION.SESSION_REF = ?
     ORDER BY
       ENDPOINT_TYPE_DEVICE.DEVICE_TYPE_ORDER`,
     [endpointTypeId, sessionId]
@@ -873,7 +888,10 @@ async function updateEndpointType(db, sessionId, endpointTypeId, changesArray) {
 /**
  * Promise to set the default attributes and clusters for a endpoint type.
  * @param {*} db
+ * @param {*} sessionId
  * @param {*} endpointTypeId
+ * @param {*} deviceTypeRef
+ * @param {*} doTransaction
  */
 async function setEndpointDefaults(
   db,
@@ -893,7 +911,20 @@ async function setEndpointDefaults(
   if (pkgs == null || pkgs.length < 1)
     throw new Error('Could not locate package id for a given session.')
 
+  let deviceTypeInfo =
+    await querySession.selectDeviceTypePackageInfoFromDeviceTypeId(
+      db,
+      deviceTypeRef
+    )
+  let endpointTypeCategory =
+    deviceTypeInfo.length > 0 ? deviceTypeInfo[0].category : null
   let packageId = pkgs[0].id
+  for (let i = 0; i < pkgs.length; i++) {
+    if (pkgs[i].category == endpointTypeCategory) {
+      packageId = pkgs[i].id
+      break
+    }
+  }
   let clusters = await queryDeviceType.selectDeviceTypeClustersByDeviceTypeRef(
     db,
     deviceTypeRef
@@ -1216,7 +1247,16 @@ async function resolveNonOptionalAndReportableAttributes(
 async function selectEndpointTypeCount(db, sessionId) {
   let x = await dbApi.dbGet(
     db,
-    'SELECT COUNT(ENDPOINT_TYPE_ID) AS CNT FROM ENDPOINT_TYPE WHERE SESSION_REF = ?',
+    `SELECT
+      COUNT(ENDPOINT_TYPE_ID) AS CNT
+    FROM
+      ENDPOINT_TYPE
+    INNER JOIN
+      SESSION_PARTITION
+    ON
+      SESSION_PARTITION.SESSION_PARTITION_ID = ENDPOINT_TYPE.SESSION_PARTITION_REF
+    WHERE
+      SESSION_PARTITION.SESSION_REF = ?`,
     [sessionId]
   )
   return x['CNT']
@@ -1243,7 +1283,11 @@ SELECT
   COUNT(ENDPOINT_TYPE_ID)
 FROM
   ENDPOINT_TYPE
-WHERE SESSION_REF = ?
+INNER JOIN
+  SESSION_PARTITION
+ON
+  SESSION_PARTITION.SESSION_PARTITION_ID = ENDPOINT_TYPE.SESSION_PARTITION_REF
+WHERE SESSION_PARTITION.SESSION_REF = ?
   AND ENDPOINT_TYPE_ID IN
       (SELECT ENDPOINT_TYPE_REF
        FROM ENDPOINT_TYPE_CLUSTER
@@ -1404,8 +1448,12 @@ JOIN
   ATTRIBUTE AS A ON ETA.ATTRIBUTE_REF = A.ATTRIBUTE_ID
 JOIN
   ENDPOINT_TYPE AS ET ON ETA.ENDPOINT_TYPE_REF = ET.ENDPOINT_TYPE_ID
+INNER JOIN
+  SESSION_PARTITION
+ON
+  ET.SESSION_PARTITION_REF = SESSION_PARTITION.SESSION_PARTITION_ID
 WHERE
-  ET.SESSION_REF = ? AND ETA.INCLUDED = 1
+  SESSION_PARTITION.SESSION_REF = ? AND ETA.INCLUDED = 1
 ORDER BY
   CLUSTER_CODE, ATTRIBUTE_CODE
   `,

src-electron/db/query-config.js

@@ -26,15 +26,15 @@ const dbMapping = require('./db-mapping')
 /**
  * Retrieves all the device types in the database.
  *
- * @export
  * @param {*} db
+ * @param {*} packageId
  * @returns Promise that resolves with the rows of device types.
  */
 async function selectAllDeviceTypes(db, packageId) {
   return dbApi
     .dbAll(
       db,
-      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS FROM DEVICE_TYPE WHERE PACKAGE_REF = ? ORDER BY DOMAIN, CODE',
+      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS, PACKAGE_REF FROM DEVICE_TYPE WHERE PACKAGE_REF = ? ORDER BY DOMAIN, CODE',
       [packageId]
     )
     .then((rows) => rows.map(dbMapping.map.deviceType))
@@ -51,7 +51,7 @@ async function selectDeviceTypeById(db, id) {
   return dbApi
     .dbGet(
       db,
-      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS FROM DEVICE_TYPE WHERE DEVICE_TYPE_ID = ?',
+      'SELECT DEVICE_TYPE_ID, DOMAIN, CODE, PROFILE_ID, NAME, DESCRIPTION, CLASS, PACKAGE_REF FROM DEVICE_TYPE WHERE DEVICE_TYPE_ID = ?',
       [id]
     )
     .then(dbMapping.map.deviceType)