Use SHA-256 instead of SHA3-256 for record hashing

Dockerfile

@@ -1,27 +1,37 @@
-FROM node:21-alpine3.17
+# Stage 1: Build Java JAR
+FROM node:slim@sha256:5ae787590295f944e7dc200bf54861bac09bf21b5fdb4c9b97aee7781b6d95a2 AS java-build
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git openjdk-17-jre-headless \
+    && git clone https://github.com/RESOStandards/web-api-commander --single-branch --depth 1 \
+    && rm -rf /var/lib/apt/lists/* /web-api-commander/.git
+WORKDIR /web-api-commander
+RUN ./gradlew --no-daemon clean jar
 
-RUN apk add --update git openjdk17-jre-headless 
+# Stage 2: Install Node dependencies
+FROM node:slim@sha256:5ae787590295f944e7dc200bf54861bac09bf21b5fdb4c9b97aee7781b6d95a2 AS node-build
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git \
+    && git clone https://github.com/RESOStandards/reso-certification-utils --single-branch --branch v3.0.0 --depth 1 \
+    && rm -rf /var/lib/apt/lists/* /reso-certification-utils/.git
+WORKDIR /reso-certification-utils
+RUN npm install
 
-RUN git clone https://github.com/RESOStandards/web-api-commander --single-branch
+# Stage 3: Final runtime image — no git or build tools, only the JRE needed at runtime
+FROM node:slim@sha256:5ae787590295f944e7dc200bf54861bac09bf21b5fdb4c9b97aee7781b6d95a2
+RUN apt-get update && apt-get install -y --no-install-recommends openjdk-17-jre-headless \
+    && rm -rf /var/lib/apt/lists/*
 
-# clean up
-RUN apk del git
-RUN rm -rf web-api-commander/.git
-RUN rm -rf /var/cache/apk/*
+COPY --from=java-build /web-api-commander /web-api-commander
+COPY --from=node-build /reso-certification-utils /reso-certification-utils
 
 ENV WEB_API_COMMANDER_PATH=/web-api-commander
+ENV RESO_SDK_PATH=/reso-certification-utils
+ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8
 ENV REFERENCE_METADATA_URL=https://services.reso.org/metadata
 ENV NODE_OPTIONS=--max-old-space-size=8192
 
-ADD ./ ./
-
-RUN echo "Web API Commander Path: ${WEB_API_COMMANDER_PATH}"
+WORKDIR ${RESO_SDK_PATH}
+RUN npm link
 
-WORKDIR ${WEB_API_COMMANDER_PATH}
-RUN ./gradlew --no-daemon clean jar
-
-WORKDIR /
-RUN npm install
-RUN npm i . -g
+RUN useradd -r -s /bin/false appuser
+USER appuser
 
 ENTRYPOINT ["reso-certification-utils"]

common.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const fs = require('fs');
 const { resolve, normalize, join } = require('path');
 const fse = require('fs-extra');
@@ -196,14 +198,14 @@ const buildRecipientEndorsementPath = ({
   if (!isValidEndorsement(endorsementName)) throw new Error(`Invalid endorsementName: ${endorsementName}`);
   if (!isValidVersion(endorsementName, version)) throw new Error(`Invalid version: ${version}`);
 
-  return join(
+  return resolve(normalize(join(
     process.cwd(),
     resultsPath,
     `${endorsementName}-${version}`,
     `${providerUoi}-${providerUsi}`,
     recipientUoi,
     currentOrArchived
-  );
+  )));
 };
 
 /**
@@ -344,11 +346,14 @@ const buildMetadataMap = ({ fields = [], lookups = [] } = {}) => {
     }
 
     if (isStringEnumeration(type)) {
-      acc[lookupName].push({ lookupValue, ddWikiUrl, isStringEnumeration: true });
+      // the standard lookup value in the Lookup Resource is represented as an annotated value
+      acc[lookupName].push({ lookupValue, standardLookupValue: annotatedLookupValue, ddWikiUrl, isStringEnumeration: true });
     } else {
       acc[lookupName].push({ lookupValue: annotatedLookupValue, legacyODataValue: lookupValue, ddWikiUrl });
     }
 
+    //if (!!annotatedLookupValue && annotatedLookupValue !== lookupValue) console.log(`lookupName: ${lookupName}, lookupValue: ${lookupValue}, annotatedLookupValue: ${annotatedLookupValue}`);
+
     STATS.numLookups++;
     return acc;
   }, {});
@@ -396,7 +401,7 @@ const buildMetadataMap = ({ fields = [], lookups = [] } = {}) => {
               acc[resourceName][fieldName].legacyODataValues = {};
             }
 
-            Object.values(lookupMap?.[type]).forEach(({ lookupValue, legacyODataValue, ddWikiUrl, isStringEnumeration }) => {
+            Object.values(lookupMap?.[type]).forEach(({ lookupValue, standardLookupValue, legacyODataValue, ddWikiUrl, isStringEnumeration }) => {
               const lookupName = parseLookupName(type);
 
               //skip legacyOData matching if we're using string enumerations
@@ -411,14 +416,20 @@ const buildMetadataMap = ({ fields = [], lookups = [] } = {}) => {
               }
 
               if (lookupValue?.length) {
-                acc[resourceName][fieldName].lookupValues[lookupValue] = {
+                const value = {
                   type,
                   lookupName,
                   lookupValue,
                   legacyODataValue,
                   ddWikiUrl,
                   isStringEnumeration
                 };
+
+                if (!!standardLookupValue) {
+                  value.standardLookupValue = standardLookupValue;
+                }
+
+                acc[resourceName][fieldName].lookupValues[lookupValue] = value;
               }
             });
           }
@@ -507,7 +518,7 @@ const parseBooleanValue = item => {
 };
 
 const createReplicationStateServiceInstance = () => {
-  const replicationStateService = require('./lib/replication/services/replication-state');
+  const replicationStateService = require('./lib/replication/services/state');
   replicationStateService.init();
   return replicationStateService;
 };

index.js

@@ -1,6 +1,7 @@
 #! /usr/bin/env node
 
-require('dotenv').config();
+const { existsSync } = require('fs');
+if (existsSync('.env')) process.loadEnvFile();
 
 const { schema, combineErrors, generateJsonSchema, validate, VALIDATION_ERROR_MESSAGES } = require('./lib/schema');
 const { restore } = require('./lib/restore');
@@ -15,7 +16,7 @@ const { DEFAULT_PAGE_SIZE } = require('./lib/replication/utils');
 //Only load commander interpreter if running from the CLI
 if (require?.main === module) {
   const { program } = require('commander');
-
+  
   /**
    * Ensure fromCli is true for anything run from the command line
    */
@@ -98,6 +99,7 @@ if (require?.main === module) {
         jsonSchemaValidation = false,
         maxPageSize,
         top,
+        outputPath,
         ...remainingOptions
       } = options;
 
@@ -109,7 +111,9 @@ if (require?.main === module) {
         jsonSchemaValidation: parseBooleanValue(jsonSchemaValidation),
         strictMode: parseBooleanValue(strictMode),
         maxPageSize: parseInt(maxPageSize) ?? undefined,
-        top: parseInt(top) ?? undefined
+        top: parseInt(top) ?? undefined,
+        outputPath,
+        shouldSaveResults: !!outputPath
       };
 
       if (bearerToken) {

lefthook.yml

@@ -0,0 +1,4 @@
+pre-commit:
+  commands:
+    lint-and-test:
+      run: eslint . --fix && npm t

lib/certification/data-dictionary/index.js

@@ -86,8 +86,7 @@ const executeCommanderMetadataTest = async ({ pathToWebApiCommander, version, pa
       join(pathToWebApiCommander, 'gradlew'),
       ['testDataDictionary', `-Dversion=${version}`, `-DpathToRESOScript=${pathToConfigFile}`],
       {
-        cwd: pathToWebApiCommander,
-        shell: true
+        cwd: pathToWebApiCommander
       }
     );
 
@@ -196,8 +195,8 @@ const runDDTests = async ({
 
   try {
     Object.assign(providerInfo, JSON.parse(fs.readFileSync(pathToConfigFile)));
-  } catch (err) {
-    handleError(`Could not read provider info! pathToConfigFile: ${pathToConfigFile}. Error: ${err}`);
+  } catch {
+    handleError(`Could not read provider info! pathToConfigFile: ${pathToConfigFile}`);
   }
 
   const { providerUoi, configs } = providerInfo;
@@ -260,7 +259,8 @@ const runDDTests = async ({
             replicationStateServiceInstance,
             fromCli,
             strictMode,
-            limit
+            limit,
+            outputPath: RECIPIENT_PATH
           });
         } catch (err) {
           handleError(err, { terminate: false });
@@ -393,7 +393,8 @@ const executeDataDictionaryTestSteps = async ({
   replicationStateServiceInstance,
   fromCli,
   strictMode = true,
-  limit
+  limit,
+  outputPath
 }) => {
   const steps =
     getDataDictionaryTestSteps({
@@ -403,7 +404,8 @@ const executeDataDictionaryTestSteps = async ({
       replicationStateServiceInstance,
       fromCli,
       strictMode,
-      limit
+      limit,
+      outputPath
     }) ?? [];
   for await (const step of steps) {
     await step();
@@ -422,7 +424,8 @@ const getDataDictionaryTestSteps = ({
   replicationStateServiceInstance,
   fromCli,
   strictMode = true,
-  limit
+  limit,
+  outputPath
 }) => {
   const testSteps = [];
 
@@ -439,7 +442,8 @@ const getDataDictionaryTestSteps = ({
     REPLICATION_STATE_SERVICE: replicationStateServiceInstance,
     fromCli,
     limit: parseInt(limit) ?? DEFAULT_LIMIT,
-    jsonSchemaValidation: !!strictMode
+    jsonSchemaValidation: !!strictMode,
+    outputPath
   };
 
   if (version === DATA_DICTIONARY_VERSIONS.v1_7) {
@@ -459,7 +463,7 @@ const getDataDictionaryTestSteps = ({
 
     testSteps.push(
       async () => {
-        const { variations } = await findVariations({ pathToMetadataReportJson, fromCli, strictMode });
+        const { variations } = await findVariations({ pathToMetadataReportJson, fromCli, strictMode, outputPath });
         if (strictMode && Object.values(variations).some(variation => variation?.length)) {
           throw new Error('Found variations during testing!');
         }

lib/certification/reso-common-format/README

@@ -0,0 +1 @@
+# TODO

lib/certification/reso-common-format/index.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const { getReferenceMetadata } = require('@reso/reso-certification-etl');
+const { generateJsonSchema, validate, combineErrors } = require('../../schema');
+const fs = require('fs/promises');
+const { exit } = require('process');
+const { DEFAULT_DD_VERSION } = require('../../../common');
+
+const runRcfTests = async ({
+  pathToRcfResults = null,
+  additionalProperties = false,
+  version = DEFAULT_DD_VERSION,
+  metadataReportJson = null
+} = {}) => {
+  const isS3Path = path => (path && path?.startsWith('s3://')) ?? false;
+
+  // 1. If pathToResults is S3, load from there. Otherwise load from local path
+  if (!!pathToRcfResults && isS3Path(pathToRcfResults)) {
+    // TODO S3
+    throw new Error('S3 not supported yet! Local files only.');
+  }
+
+  try {
+    // generate
+    const schema = await generateJsonSchema({
+      additionalProperties,
+      metadataReportJson: metadataReportJson ?? getReferenceMetadata(version)
+    });
+
+    // validate
+    const payload = JSON.parse(await fs.readFile(pathToRcfResults));
+
+    let errorMap = {};
+
+    errorMap = validate({
+      version: '2.0',
+      jsonPayload: payload,
+      errorMap,
+      jsonSchema: schema,
+      resourceName: 'Property',
+      disableKeys: true
+    });
+
+    const errorReport = combineErrors(errorMap);
+    console.log(JSON.stringify(errorReport, null, 2));
+
+    // const errorJson = {
+    //   description: 'RESO Common Format Error Report'
+    // };
+
+  } catch (err) {
+    console.error(err);
+    exit(true);
+  }
+
+  // 2. Process RCF files with schema validation
+
+  // 3. Generate data availability report
+
+  // 4. Generate metadata report
+
+  // 5. Run variations check - if it fails, post to Cert API with failed status and variations report
+
+  // 6. If variations check passes, save metadata and availability reports to S3, then push to Cert API
+
+  // 7. For any path that is from S3, delete the raw data from S3 after the request is finished,
+  // whether it completed successfully or not
+};
+
+module.exports = {
+  runRcfTests
+};

lib/certification/upi/index.js

@@ -56,7 +56,7 @@ const runUpiTests = async ({
     }
 
     upiData.forEach(item => {
-      const { UniversalParcelId, ParcelNumber: suppliedParcelNumber } = item;
+      const { UniversalParcelId, Country: suppliedCountry, CountrySubdivision: suppliedCountrySubdivision, ParcelNumber: suppliedParcelNumber } = item;
 
       if (!UniversalParcelId) {
         throw new Error('UniversalParcelId not present in data!');
@@ -67,7 +67,7 @@ const runUpiTests = async ({
       }
 
       try {
-        const { CountrySubdivision: parsedCountrySubdivision, ParcelNumber: parsedParcelNumber } = parseUpi(UniversalParcelId);
+        const { Country: parsedCountry, CountrySubdivision: parsedCountrySubdivision, ParcelNumber: parsedParcelNumber } = parseUpi(UniversalParcelId);
 
         if (!validateCountrySubdivision(parsedCountrySubdivision, COUNTRY_SUBDIVISION_CACHE)) {
           invalidIds.push({ upi: UniversalParcelId, error: `Invalid country subdivision '${parsedCountrySubdivision}'` });
@@ -77,6 +77,14 @@ const runUpiTests = async ({
           //console.error(`Parsed parcel number '${parsedParcelNumber}' differs from the one provided in the payload: '${suppliedParcelNumber}'`);
           invalidIds.push({ upi: parsedParcelNumber, error: 'Parsed UPI mismatch with UPI data' });
         }
+
+        if (suppliedCountry && suppliedCountry !== parsedCountry) {
+          invalidIds.push({ upi: UniversalParcelId, error: 'Parsed UPI mismatch with UPI data' });
+        }
+
+        if (suppliedCountrySubdivision && suppliedCountrySubdivision !== parsedCountrySubdivision) {
+          invalidIds.push({ upi: UniversalParcelId, error: 'Parsed UPI mismatch with UPI data' });
+        }
       } catch (err) {
         invalidIds.push({ upi: UniversalParcelId, error: err?.message ?? 'Parsing Error' });
       }