Time varying conversions

package.json

@@ -32,7 +32,7 @@
 		"refreshDailyReadingViews": "node -e 'require(\"./src/server/services/refreshReadingViews\").refreshReadingViews()'",
 		"refreshHourlyReadingViews": "node -e 'require(\"./src/server/services/refreshHourlyReadingViews\").refreshHourlyReadingViews()'",
 		"refreshGroupsDeepMeters": "node -e 'require(\"./src/server/services/refreshGroupsDeepMetersView\").refreshGroupsDeepMetersView()'",
-		"updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikAndViews()'",
+		"updateCikAndViews": "node -e 'require(\"./src/server/services/graph/redoCik.js\").updateCikVaryAndViews()'",
 		"obvius:showConfigfiles": "node ./src/server/services/obvius/showConfigfiles.js",
 		"obvius:purgeConfigfiles": "node ./src/server/services/obvius/purgeConfigfiles.js",
 		"generateFourDayTestingData": "node -e 'require(\"./src/server/data/automatedTestingData\").generateFourDayTestingData()'",

src/client/app/types/redux/ciks.ts

@@ -5,6 +5,4 @@
 export interface CikData {
 	meterUnitId: number;
 	nonMeterUnitId: number;
-	slope: number;
-	intercept: number;
 }

src/server/data/automatedTestingData.js

@@ -8,11 +8,11 @@
 
 const { generateSine, generateCosine } = require('./generateTestingData');
 const Unit = require('../models/Unit');
+const Group = require('../models/Group');
 const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData');
 const { getConnection } = require('../db');
-const { redoCik } = require('../services/graph/redoCik');
+const { redoCikVary } = require('../services/graph/redoCik');
 const { refreshAllReadingViews } = require('../services/refreshAllReadingViews');
-const fs = require('fs').promises;
 
 // Define the start and end date for data generation.
 const DEFAULT_OPTIONS = {
@@ -1335,18 +1335,20 @@ async function insertSpecialUnitsConversionsMetersGroups() {
 	await insertSpecialConversions(conn);
 	// Recreate the Cik entries since changed units/conversions.
 	// Do now since needed to insert meters with suffix units.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	// Generate the mathematical test data needed.
 	console.log(`Start loading each set of test data into OED meters, may take minutes):\n`);
 	// This is very fast so wait since simpler and easier to see if this part fails.
 	await insertMeters(specialMeters, conn);
 	// Now do the large dataset generation.
 	await testData();
 	// Recreate the Cik entries since changed meters.
-	await redoCik(conn);
-	// Refresh the readings since added new ones.
-	await refreshAllReadingViews();
+	await redoCikVary(conn);
 	await insertGroups(specialGroups, conn);
+	// Refresh groups deep meters view after adding new groups.
+	await Group.refreshGroupsDeepMetersView(conn);
+	// Refresh the readings since added new ones & groups.
+	await refreshAllReadingViews();
 }
 
 /*

src/server/data/websiteData.js

@@ -11,7 +11,7 @@
 
 const Unit = require('../models/Unit');
 const Group = require('../models/Group');
-const { redoCik } = require('../services/graph/redoCik');
+const { redoCikVary } = require('../services/graph/redoCik');
 const { refreshAllReadingViews } = require('../services/refreshAllReadingViews');
 const { getConnection } = require('../db');
 const { insertUnits, insertStandardUnits, insertConversions, insertStandardConversions, insertMeters, insertGroups } = require('../util/insertData');
@@ -753,12 +753,12 @@ async function insertWebsiteData() {
 	await insertConversions(conversions, conn);
 	// Recreate the Cik entries since changed units/conversions.
 	// Do now since needed to insert meters with suffix units.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	console.log(`Start loading each set of test data into OED meters (${meters.length} files of varying length, may take minutes):`);
 	// await Meter.insertMany(meters, conn);
     await insertMeters(meters, conn);
 	// Recreate the Cik entries since changed meters.
-	await redoCik(conn);
+	await redoCikVary(conn);
 	await insertGroups(groups, conn);
 	// Refresh groups deep meters view after adding new groups.
 	await Group.refreshGroupsDeepMetersView(conn);

src/server/migrations/2.0.0-3.0.0/index.js

@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+const database = require('../../models/database');
+const sqlFile = database.sqlFile;
+
+module.exports = {
+    fromVersion: '2.0.0',
+    toVersion: '3.0.0',
+    up: async db => {
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/drop_old_views.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_reading_views.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql'));
+        await db.none(sqlFile('../migrations/2.0.0-3.0.0/sql/readings/create_function_get_compare_readings.sql'));
+    }
+};

src/server/migrations/2.0.0-3.0.0/sql/cik/alter_cik_table.sql

@@ -0,0 +1,10 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+--Add the new columns with appropriate infinity defaults
+ALTER TABLE cik
+	DROP COLUMN slope,
+	DROP COLUMN intercept
+;

src/server/migrations/2.0.0-3.0.0/sql/readings/create_function_get_3d_readings.sql

@@ -0,0 +1,288 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+-- By indexing both columns together, the database can efficiently handle queries that involve both meter_id and time_interval.
+-- Created to support the usage of the view by 3d.
+-- TODO verify helps with newer views.
+CREATE INDEX IF NOT EXISTS idx_meter_hourly_readings_unit_meter_time
+ON meter_hourly_readings_unit (meter_id, lower(time_interval));
+-- TODO How does this relate to index for same view in create_reading_views? Are both needed?
+CREATE INDEX if not exists idx_two_group_hourly_readings_unit
+ON group_hourly_readings_unit (group_id, graphic_unit_id, lower(time_interval));
+
+/*
+This takes tsrange_to_shrink which is the requested time range to plot and makes sure it does
+not exceed the start/end times for the readings in the supplied meter. This can be an issue, in particular,
+because infinity is used to indicate to graph all readings. This version does it to the nearest
+day by using the day reading view and is used by 3D readings which only allow days and a single meter.
+ */
+CREATE OR REPLACE FUNCTION shrink_tsrange_to_meter_readings_by_day(tsrange_to_shrink TSRANGE, meter_id_desired INTEGER)
+	RETURNS TSRANGE
+AS $$
+DECLARE
+	readings_max_tsrange TSRANGE;
+BEGIN
+	SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange
+	FROM meter_daily_readings_unit
+	where meter_id = meter_id_desired;
+	RETURN tsrange_to_shrink * readings_max_tsrange;
+END;
+$$ LANGUAGE 'plpgsql';
+
+/* Similar to meter version but for a group */
+CREATE OR REPLACE FUNCTION shrink_tsrange_to_group_readings_by_day(tsrange_to_shrink TSRANGE, group_id_desired INTEGER)
+	RETURNS TSRANGE
+AS $$
+DECLARE
+	readings_max_tsrange TSRANGE;
+BEGIN
+	SELECT tsrange(min(lower(time_interval)), max(upper(time_interval))) INTO readings_max_tsrange
+	FROM group_daily_readings_unit
+	where group_id = group_id_desired;
+	RETURN tsrange_to_shrink * readings_max_tsrange;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- Determines the spacing between 3D points. It uses the lowest one for all meters passed
+-- that is valid.
+CREATE OR REPLACE FUNCTION reading_interval_3d (
+	-- The desired meter ids.
+	IN meter_ids_requested INTEGER[],
+	-- The number of hours in each reading requested
+	IN reading_length_hours INTEGER,
+	-- The number of hours in each reading determined
+	OUT reading_length_hours_use INTEGER,
+	-- The number of hours in each reading determined as an interval
+	OUT reading_length_interval INTERVAL
+)
+AS $$
+DECLARE
+	-- The meter frequency from all meters.
+ 	meter_frequency INTERVAL;
+	-- The meter frequency rounded up to a whole number of hours.
+   	meter_frequency_hour_up INTEGER;
+	-- The larger of the meter value and the argument sent.
+	max_frequency INTEGER;
+BEGIN
+	-- Get the smallest reading frequency for all meters requested.
+	SELECT min(reading_frequency) INTO meter_frequency
+	FROM (meters m
+	INNER JOIN unnest(meter_ids_requested) meters(id) ON m.id = meters.id);
+  	-- Get the seconds in the frequency from epoch, /3600 To get hours and then round up to a whole number of hours.
+	meter_frequency_hour_up := CEIL((SELECT * FROM EXTRACT(EPOCH FROM meter_frequency)) / 3600);
+	-- Use the hours that is the largest of the request and the meter values.
+	max_frequency := GREATEST(meter_frequency_hour_up, reading_length_hours);
+	-- The value used must be a divisor of 24 or greater than 12.
+	IF (max_frequency = 5) THEN
+		reading_length_hours_use := 6;
+	ELSIF (max_frequency = 7) THEN
+		reading_length_hours_use := 8;
+	ELSIF (max_frequency > 8 AND max_frequency < 12) THEN
+		reading_length_hours_use := 12;
+	ELSE
+		reading_length_hours_use := max_frequency;
+	END IF;
+	-- Hours per reading determined returned as an interval.
+	reading_length_interval := (reading_length_hours_use::TEXT || ' hour')::INTERVAL;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- Gets meters graphing data for 3D graphic by returning points that span the requested
+-- length of time over the days requested.
+-- New meter_3d_readings_unit function that uses new meter_hourly_readings_unit view.
+CREATE OR REPLACE FUNCTION meter_3d_readings_unit (
+	-- The desired meter ids. It is normally a single value for a 3D graphic.
+	-- TODO Should the array be changed to a single value as with group? Need to be sure client never asks for multiple.
+	meter_ids_requested INTEGER[],
+	-- The desired graphic unit of the returned data
+	graphic_unit_id_requested INTEGER,
+	-- The start and end time for the data to return
+	start_stamp TIMESTAMP,
+	end_stamp TIMESTAMP,
+	-- The number of hours in each reading requested
+	reading_length_hours INTEGER
+)
+	RETURNS TABLE(meter_id INTEGER, reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP)
+AS $$
+DECLARE
+	-- Holds the range of dates for returned data that fits the actual data.
+	requested_range TSRANGE;
+	-- The number of hours in each reading determined as an interval
+	reading_length_interval INTERVAL;
+	-- Which index of the meter_id array you are currently working on.
+	current_meter_index INTEGER := 1;
+	-- The id of the meter index working on
+	current_meter_id INTEGER;
+	-- The meter frequency from all meters.
+ 	meter_frequency INTERVAL;
+	-- The meter frequency rounded up to a whole number of hours.
+   	meter_frequency_hour_up INTEGER;
+	-- The larger of the meter value and the argument sent.
+	max_frequency INTEGER;
+	-- The number of hours in each reading determined
+	reading_length_hours_use INTEGER;
+BEGIN
+	-- Find the correct number of hours per reading returned.
+	SELECT * from reading_interval_3d(meter_ids_requested, reading_length_hours) into reading_length_hours_use, reading_length_interval;
+
+	-- Loop over all meters.
+	WHILE current_meter_index <= cardinality(meter_ids_requested) LOOP
+		-- ID of the current meter in loop
+		current_meter_id := meter_ids_requested[current_meter_index];
+
+		-- Get the range of days requested by calling shrink_tsrange_to_meter_readings_by_day.
+		-- First make requested range only be full days by dropping any partial days at start/end.
+		requested_range := shrink_tsrange_to_meter_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), current_meter_id);
+
+		IF (reading_length_hours_use <= 12) THEN
+			-- Need to generate_series to group the desired hours together
+			RETURN QUERY
+				-- The readings are rates in the hourly table so want to average not sum so
+				-- work for quantity, flow & raw.
+				-- The time starts at the time of the generated sequence and ends at the length
+				-- of each block later. This is the same as the start time of the next value
+				-- in the sequence (except last one).
+				SELECT
+					-- Modified to retrieve converted hourly readings from the materialized view.
+					mhr.meter_id as meter_id,
+					AVG(mhr.reading_rate) as reading_rate,
+					hours.hour AS start_timestamp,
+					hours.hour + reading_length_interval  AS end_timestamp
+				-- This is the series that starts at the beginning of the desired days,
+				-- ends at the end of the desired days and steps by the desired interval.
+				-- You need to subtract from the last interval for the end since generate_series
+				-- is inclusive.
+				FROM (
+					SELECT hour
+					FROM generate_series(
+						lower(requested_range),
+						upper(requested_range) - reading_length_interval,
+						reading_length_interval
+					) hours(hour)
+				) hours(hour),
+				-- Also need the values in the meter hourly table.
+				meter_hourly_readings_unit mhr
+				-- Only want the desired meter
+				WHERE mhr.meter_id = current_meter_id
+				-- Only want the desired graphing unit
+				AND mhr.graphic_unit_id = graphic_unit_id_requested
+				-- Only want readings that lie within this slice of the desired data
+				AND lower(mhr.time_interval) >= hours.hour
+				AND upper(mhr.time_interval) <= hours.hour + reading_length_interval
+				-- ensures that the start of the reading time intervals does not exceed the end of the current generated interval
+				AND lower(mhr.time_interval) <= hours.hour + reading_length_interval
+				-- Group by the start time of the generated series since all points in
+				-- the desired slice have the same start time for the series.
+				-- Also group by the meter_id since Postgres wants and desired for graphing
+				GROUP BY hours.hour, mhr.meter_id
+				-- Time sort by the meter and start time for graphing.
+				ORDER BY mhr.meter_id, hours.hour
+			;
+		ELSE
+			-- The reading rate is more than 12 so return a single row with dummy values that easy to detect.
+			-- The end time differs from the start time by the meter reading frequency or min one for groups.
+			-- This means the meter reading frequency is too long for a 3D graphic.
+			RETURN QUERY
+				SELECT -999, -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval
+			;
+		END IF;
+
+		-- Go to the next meter
+		current_meter_index := current_meter_index + 1;
+	END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
+/* Gets group graphing data for 3D graphic by returning points that span the requested
+  length of time over the days requested.
+*/
+CREATE OR REPLACE FUNCTION group_3d_readings_unit (
+	--Desire group ID
+	--For 3D graphics, users will only be able to select 1 group to graph.
+	group_id_requested INTEGER,
+	-- The desired graphic unit of the returned data
+	graphic_unit_id_requested INTEGER,
+	-- The start and end time for the data to return
+	start_stamp TIMESTAMP,
+	end_stamp TIMESTAMP,
+	-- The number of hours in each reading requested
+	reading_length_hours INTEGER
+)
+	RETURNS TABLE(reading_rate FLOAT, start_timestamp TIMESTAMP, end_timestamp TIMESTAMP)
+AS $$
+DECLARE
+	-- Holds the range of dates for returned data that fits the actual data.
+	requested_range TSRANGE;
+	--Holds the desired meter IDs in order to call meter_3d_readings_unit in the query below. 
+	meter_ids INTEGER[];
+	-- The number of hours in each reading determined
+	reading_length_hours_use INTEGER;
+	-- The number of hours in each reading determined as an interval
+	reading_length_interval INTERVAL;
+BEGIN
+	--Get all the meter IDS that will be included in the group being requested.
+	SELECT array_agg(DISTINCT gdm.meter_id) INTO meter_ids
+	FROM groups_deep_meters gdm
+	WHERE group_id = group_id_requested;
+
+	-- Find the correct number of hours per reading returned.
+	SELECT * from reading_interval_3d(meter_ids, reading_length_hours) into reading_length_hours_use, reading_length_interval;
+	-- Get the range of days requested by calling shrink_tsrange_to_group_readings_by_day.
+	-- First make requested range only be full days by dropping any partial days at start/end.
+	requested_range := shrink_tsrange_to_group_readings_by_day(tsrange(date_trunc_up('day', start_stamp), date_trunc('day', end_stamp)), group_id_requested);
+
+	IF (reading_length_hours_use <= 12) THEN
+		-- Need to generate_series to group the desired hours together
+		RETURN QUERY
+			-- The readings are rates in the hourly table so want to average not sum so
+			-- work for quantity, flow & raw.
+			-- The time starts at the time of the generated sequence and ends at the length
+			-- of each block later. This is the same as the start time of the next value
+			-- in the sequence (except last one).
+			SELECT
+				AVG(ghr.reading_rate) as reading_rate,
+				hours.hour AS start_timestamp,
+				hours.hour + reading_length_interval  AS end_timestamp
+				-- This is the series that starts at the beginning of the desired days,
+				-- ends at the end of the desired days and steps by the desired interval.
+				-- You need to subtract from the last interval for the end since generate_series
+				-- is inclusive.
+			FROM (
+				SELECT hour
+				FROM generate_series(
+					lower(requested_range),
+					upper(requested_range) - reading_length_interval,
+					reading_length_interval
+				) hours(hour)
+			) hours(hour),
+			-- Also need the values in the group hourly table.
+			group_hourly_readings_unit ghr
+			-- Only want the desired meter
+			WHERE ghr.group_id = group_id_requested
+			-- Only want the desired graphing unit
+			AND ghr.graphic_unit_id = graphic_unit_id_requested
+			-- Only want readings that lie within this slice of the desired data
+			AND lower(ghr.time_interval) >= hours.hour
+			AND upper(ghr.time_interval) <= hours.hour + reading_length_interval
+			-- ensures that the start of the reading time intervals does not exceed the end of the current generated interval
+			AND lower(ghr.time_interval) <= hours.hour + reading_length_interval
+			-- Group by the start time of the generated series since all points in
+			-- the desired slice have the same start time for the series.
+			GROUP BY hours.hour
+			-- Time sort by the start time for graphing.
+			ORDER BY hours.hour
+		;
+	ELSE
+		-- The reading rate is more than 12 so return a single row with dummy values that easy to detect.
+		-- The end time differs from the start time by the meter reading frequency or min one for groups.
+		-- This means the meter reading frequency is too long for a 3D graphic.
+		RETURN QUERY
+			SELECT -999::FLOAT, '1900-01-01 00:00:00'::TIMESTAMP, '1900-01-01 00:00:00'::TIMESTAMP + reading_length_interval
+		;
+	END IF;	
+END;
+$$ LANGUAGE 'plpgsql';