Bug fixes for commercial post-processing helper script and associated files

scripts/energy savings/commercial/DEER_weights_extraction.py

@@ -36,17 +36,28 @@
 #filter for BldgType=Com weights
 flag_weightID = df['weightID']=='Com'
 flag_Sector = (df['Sector']=='Com')|(df['Sector']=='Ind')
+# 2025-09-29 Nicholas Fette (Solaris Technical)
+# Filter table by version AND bump version from DEER2020 to DEER2026
+flag_Version = (df['Version']=='DEER2026')
 
-df_bldgtype_com_wts = df[flag_weightID & flag_Sector]
+df_bldgtype_com_wts = df[flag_weightID & flag_Sector & flag_Version]
 # %%
 #condense table
 wts_com_bldg = df_bldgtype_com_wts[['BldgType', 'BldgVint','BldgLoc','weight']]
 wts_com_bldg = wts_com_bldg.assign(PA='Any')
 # %%
 #rearrange/rename table
-wts_com_bldg.rename(columns={'weight':'sum_bldg'}, inplace=True)
-wts_com_bldg_final = wts_com_bldg[['PA','BldgType', 'BldgVint','BldgLoc','sum_bldg']]
-wts_com_bldg_final_sorted = wts_com_bldg_final.sort_values(by=['BldgType', 'BldgVint','BldgLoc'])
+# 2025-09-29 Nicholas Fette (Solaris Technical)
+# Update renamed columns to be consistent with scripts\energy savings\commercial\support_and_setup_tables\create_wts_com_bldg.sql
+wts_com_bldg = wts_com_bldg.rename(columns={
+    'PA':'pa',
+    'BldgType':'bldgtype',
+    'BldgVint':'era',
+    'BldgLoc':'bldgloc',
+    'weight':'sum_bldg'
+    })
+wts_com_bldg_final = wts_com_bldg[['pa', 'bldgtype', 'era', 'bldgloc', 'sum_bldg']]
+wts_com_bldg_final_sorted = wts_com_bldg_final.sort_values(by=['bldgtype', 'era', 'bldgloc'])
 
 #%%
 #export wts_com_bldg

scripts/energy savings/commercial/postgres-template-com.psql

@@ -1,8 +1,8 @@
 /*
 postgres-template-com.psql
-Nicholas Fette, created 2023-01-05, modeified 2024-04-22
+Nicholas Fette, created 2023-01-05, modified 2025-09-29
 
-Automate workflow in PostgreSQL:
+Automate DEER weighted average processing workflow in PostgreSQL:
 Set up support tables, import results data , and run all the post-processing scripts.
 Warning: deletes the results data currently in the designated database schema.
 
@@ -31,74 +31,115 @@ Within an open interactive psql session:
 */
 
 \set ON_ERROR_STOP on
-
--- Set variables for this PSQL session.
-\set DEERROOT '../../'
-\set SCRIPTS :'DEERROOT' 'scripts/'
-\set SCHEMAFILES :'DEERROOT' 'schema/'
-\set ENERGYSAVINGSCOM :'SCRIPTS' 'energy savings/commercial/'
-\set STUDY './'
+\timing on
+--\set ECHO all
+-- Formatting control codes 
+\set H0 '\033[0m'
+\set H1 '\n\033[4;1m'
+\set H2 '\033[1m--'
+\set H3 '\033[3;32m'
+
+-- Set folder paths for this PSQL session.
+-- Folder paths may be absolute or relative to this script.
+\set DEERROOT '../../../'
+--\set DEERROOT 'C:/DEER-Prototypes-EnergyPlus/'
+-- These commands build folder path strings by concatenating previously defined variables. 
+\set SCRIPTS :DEERROOT 'scripts/'
+\set SCHEMAFILES :SCRIPTS 'energy savings/commercial/support_and_setup_tables/'
+\set ENERGYSAVINGSCOM :SCRIPTS 'energy savings/commercial/'
+\set STUDY 'results/'
 \set SCHEMANAME 'MC_results_database'
 \echo 'Assuming scripts are located in ' :'ENERGYSAVINGSCOM'
 \echo 'Assuming measure results are located in subfolders of ' :'STUDY'
 \echo 'Assuming schema name ' :'SCHEMANAME'
 
-\echo Creating the schema and support tables.
+\echo :H1'[0/7] Creating the schema and set search path.':H0
 CREATE SCHEMA IF NOT EXISTS :"SCHEMANAME";
 SET search_path TO :"SCHEMANAME";
 
-\echo Loading support tables
+\echo :H1'[1/7] Loading support tables':H0
 
-\echo peakperspec
-\i :'SCHEMAFILES' 'create_peakperspec.sql'
-truncate "peakperspec";
-\copy "peakperspec" FROM :'ENERGYSAVINGSCOM' 'peakperspec.csv' WITH (FORMAT csv, HEADER MATCH)
+-- Note that PSQL commands with a filename as an argument, like \copy and \i,
+-- have some limitations and the following can cause errors:
+--   Passing a file path variable containing spaces.
+--   Attempting to concatenate two strings into a filename within the body of the command.
+-- To get around this limitation and to support debugging, we can define the
+-- filename or the entire command including its arguments as a variable, echo
+-- it for debugging, and then execute it.   
 
-\echo wts_com_bldg
-\i :'SCHEMAFILES' 'create_wts_com_bldg.sql'
+\echo :H2'Support table peakperspec':H0
+\set file :SCHEMAFILES 'create_peakperspec.sql' \set command '\\i ' :'file'
+\echo :H3:command:H0
+:command
+truncate "peakperspec";
+\set file :ENERGYSAVINGSCOM 'peakperspec.csv'
+\set command '\\copy "peakperspec" FROM ' :'file' ' WITH (FORMAT csv, HEADER MATCH)'
+\echo :H3:command:H0
+:command
+
+\echo :H2'Support table wts_com_bldg':H0
+\set file :SCHEMAFILES 'create_wts_com_bldg.sql' \set command '\\i ' :'file' 
+\echo :H3:command:H0
+:command
 truncate "wts_com_bldg";
-\copy "wts_com_bldg" FROM :'ENERGYSAVINGSCOM' 'wts_com_bldg.csv' WITH (FORMAT csv, HEADER MATCH)
-
-\echo Placeholder tables for imported data
-
-\i :'SCHEMAFILES' 'create_current_msr_mat.sql'
-\i :'SCHEMAFILES' 'create_sim_annual.sql'
-\i :'SCHEMAFILES' 'create_sim_hourly_wb.sql'
-
-\echo Import current_msr_mat
+\set file :ENERGYSAVINGSCOM 'wts_com_bldg.csv'
+\set command '\\copy "wts_com_bldg" FROM ' :'file' ' WITH (FORMAT csv, HEADER MATCH)'
+\echo :H3:command:H0
+:command
+
+\echo :H1'[2/7] Creating placeholder tables for imported data':H0
+
+\set file :SCHEMAFILES 'create_current_msr_mat.sql' \set command '\\i ' :'file'
+\echo :H3:command:H0
+:command
+\set file :SCHEMAFILES 'create_sim_annual.sql' \set command '\\i ' :'file'
+\echo :H3:command:H0
+:command
+\set file :SCHEMAFILES 'create_sim_hourly_wb.sql' \set command '\\i ' :'file'
+\echo :H3:command:H0
+:command
+
+\echo :H1'[3/7] Importing data files generated by Com.py':H0
+
+\echo :H2'Import current_msr_mat':H0
 truncate "current_msr_mat";
--- Add as many lines as you have individual folders within the measure.
-\copy "current_msr_mat" FROM :'STUDY' 'SWXX000-00 Measure Name_Ex/current_msr_mat.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "current_msr_mat" FROM :'STUDY' 'SWXX000-00 Measure Name_New/current_msr_mat.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "current_msr_mat" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_Ex/current_msr_mat.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "current_msr_mat" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_New/current_msr_mat.csv' WITH (FORMAT csv, HEADER MATCH)
+\set file :STUDY 'current_msr_mat.csv'
+\set command '\\copy "current_msr_mat" FROM ' :'file' ' WITH (FORMAT csv, HEADER MATCH)'
+\echo :H3:command:H0
+:command
 
-\echo Import sim_annual
+\echo :H2'Import sim_annual':H0
 truncate "sim_annual";
--- Add as many lines as you have individual folders within the measure.
-\copy "sim_annual" FROM :'STUDY' 'SWXX000-00 Measure Name_Ex/sim_annual.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_annual" FROM :'STUDY' 'SWXX000-00 Measure Name_New/sim_annual.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_annual" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_Ex/sim_annual.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_annual" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_New/sim_annual.csv' WITH (FORMAT csv, HEADER MATCH)
+\set file :STUDY 'sim_annual.csv'
+\set command '\\copy "sim_annual" FROM ' :'file' ' WITH (FORMAT csv, HEADER MATCH)'
+\echo :H3:command:H0
+:command
 
-\echo Import sim_hourly_wb
+\echo :H2'Import sim_hourly_wb':H0
 truncate "sim_hourly_wb";
--- Add as many lines as you have individual folders within the measure.
-\copy "sim_hourly_wb" FROM :'STUDY' 'SWXX000-00 Measure Name_Ex/sim_hourly_wb.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_hourly_wb" FROM :'STUDY' 'SWXX000-00 Measure Name_New/sim_hourly_wb.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_hourly_wb" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_Ex/sim_hourly_wb.csv' WITH (FORMAT csv, HEADER MATCH)
-\copy "sim_hourly_wb" FROM :'STUDY' 'SWXX000-00 Measure Name_Htl_New/sim_hourly_wb.csv' WITH (FORMAT csv, HEADER MATCH)
-
-\echo Run step P1
-\i :'ENERGYSAVINGSCOM' 'P1-Create-sim_peakper_2023.sql'
-\echo Run step P2
-\i :'ENERGYSAVINGSCOM' 'P2-Calc_MsrImpacts_2023.sql'
-\echo Run step P3
-\i :'ENERGYSAVINGSCOM' 'P3-Bldg_wt_Mult.sql'
-\echo Run step P4
-\i :'ENERGYSAVINGSCOM' 'P4-SumBldg.sql'
-\echo Run step P5
-\i :'ENERGYSAVINGSCOM' 'P5-roundDEER.sql'
-
-\echo Write file meas_impacts_2024_com.csv (in the current directory).
+\set file :STUDY 'sim_hourly_wb.csv'
+\set command '\\copy "sim_hourly_wb" FROM ' :'file' ' WITH (FORMAT csv, HEADER MATCH)'
+\echo :H3:command:H0
+:command
+
+\echo :H1'[4/7] Computing peak demand from hourly data':H0
+
+\echo :H2'Run script P1':H0
+\set file :ENERGYSAVINGSCOM 'P1-Create-sim_peakper_2023.sql' \i :file
+
+\echo :H1'[5/7] Calculating measure energy impacts':H0
+
+\echo :H2'Run script P2 (merge base case and measure case, normalize)':H0
+\set file :ENERGYSAVINGSCOM 'P2-Calc_MsrImpacts_2023.sql' \i :file
+\echo :H2'Run script P3 (sector weighted average, step 1 of 2)':H0
+\set file :ENERGYSAVINGSCOM 'P3-Bldg_wt_Mult.sql' \i :file
+\echo :H2'Run script P4 (sector weighted average, step 2 of 2)':H0
+\set file :ENERGYSAVINGSCOM 'P4-SumBldg.sql' \i :file
+\echo :H2'Run script P5 (rounding)':H0
+\set file :ENERGYSAVINGSCOM 'P5-roundDEER.sql' \i :file
+
+\echo :H1'[6/7] Writing output files':H0
+\echo :H2'Write file meas_impacts_2024_com.csv (in the current directory).':H0
 \copy "meas_impacts_2024_com" to 'meas_impacts_2024_com.csv' WITH (FORMAT csv, HEADER)
+
+\echo :H1'[7/7] Done':H0

scripts/energy savings/commercial/support_and_setup_tables/create_current_msr.mat.sql → scripts/energy savings/commercial/support_and_setup_tables/create_current_msr_mat.sql

@@ -1,8 +1,12 @@
--- Table: MC_results_database.current_msr_mat
+-- 2025-09-29 Nicholas Fette (Solaris Technical): Don't hard-code the search_path.
 
--- DROP TABLE "MC_results_database".current_msr_mat;
+-- Uncomment this line if you need to run this query manually.
+--SET search_path TO "MC_results_database";
 
-CREATE TABLE IF NOT EXISTS "MC_results_database".current_msr_mat
+-- Uncomment this line if you need to redefine this table.
+-- DROP TABLE IF EXISTS current_msr_mat;
+
+CREATE TABLE IF NOT EXISTS current_msr_mat
 (
     "MeasureID" text COLLATE pg_catalog."default",
     "BldgType" text COLLATE pg_catalog."default",
@@ -18,9 +22,12 @@ CREATE TABLE IF NOT EXISTS "MC_results_database".current_msr_mat
     "MsrSizingID" text COLLATE pg_catalog."default",
     "SizingSrc" text COLLATE pg_catalog."default",
     "EU_HrRepVar" text COLLATE pg_catalog."default",
-    "NormUnit" text COLLATE pg_catalog."default"
+    "NormUnit" text COLLATE pg_catalog."default",
+    -- 2025-09-29 Nicholas Fette (Solaris Technical)
+    -- Cause explicit error if user tries to load duplicate data. This helps avoid accidental mistakes.
+    PRIMARY KEY ("MeasureID","BldgType","BldgVint","BldgLoc","BldgHVAC","tstat","NormUnit")
 )
 WITH (
     OIDS = FALSE
 )
-TABLESPACE pg_default;
+TABLESPACE pg_default;

scripts/energy savings/commercial/support_and_setup_tables/create_peakperspec.sql

@@ -1,55 +1,24 @@
-/*
-Navicat PGSQL Data Transfer
-
-Source Server         : DEER_Local
-Source Server Version : 90405
-Source Host           : localhost:5432
-Source Database       : deer2020
-Source Schema         : comairac
-
-Target Server Type    : PGSQL
-Target Server Version : 90405
-File Encoding         : 65001
-
-Date: 2018-08-23 13:31:00
-*/
-
 
 -- ----------------------------
 -- Table structure for peakperspec
 -- ----------------------------
-SET search_path TO "MC_results_database";
-DROP TABLE IF EXISTS "peakperspec";
-CREATE TABLE "peakperspec" (
+-- 2025-09-29 Nicholas Fette (Solaris Technical): Don't hard-code the search_path.
+
+-- Uncomment this line if you need to run this query manually.
+--SET search_path TO "MC_results_database";
+
+-- Uncomment this line if you need to redefine this table.
+--DROP TABLE IF EXISTS "peakperspec";
+
+CREATE TABLE IF NOT EXISTS "peakperspec" (
 "BldgLoc" text COLLATE "default",
 "PkDay" text COLLATE "default",
-"PkHr" text COLLATE "default"
+"PkHr" text COLLATE "default",
+-- 2025-09-29 Nicholas Fette (Solaris Technical)
+-- Cause explicit error if user tries to load duplicate data. This helps avoid accidental mistakes.
+PRIMARY KEY ("BldgLoc")
 )
 WITH (OIDS=FALSE)
-
 ;
-
--- ----------------------------
--- Records of peakperspec
--- Updated for CZ2022 weather data, 3/29/2021
--- ----------------------------
-INSERT INTO "peakperspec" VALUES ('CZ01', '238', '0');
-INSERT INTO "peakperspec" VALUES ('CZ02', '238', '0');
-INSERT INTO "peakperspec" VALUES ('CZ03', '238', '0');
-INSERT INTO "peakperspec" VALUES ('CZ04', '238', '0');
-INSERT INTO "peakperspec" VALUES ('CZ05', '259', '0');
-INSERT INTO "peakperspec" VALUES ('CZ06', '245', '0');
-INSERT INTO "peakperspec" VALUES ('CZ07', '245', '0');
-INSERT INTO "peakperspec" VALUES ('CZ08', '245', '0');
-INSERT INTO "peakperspec" VALUES ('CZ09', '244', '0');
-INSERT INTO "peakperspec" VALUES ('CZ10', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ11', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ12', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ13', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ14', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ15', '180', '0');
-INSERT INTO "peakperspec" VALUES ('CZ16', '224', '0');
-
--- ----------------------------
--- Alter Sequences Owned By 
--- ----------------------------
+-- 2025-09-29 Nicholas Fette (Solaris Technical)
+-- Don't import the data via the schema definition, but instead use the CSV file after creating the table structure.

scripts/energy savings/commercial/support_and_setup_tables/create_sim_annual.sql

@@ -1,4 +1,12 @@
-CREATE TABLE IF NOT EXISTS test_schema.sim_annual
+-- 2025-09-29 Nicholas Fette (Solaris Technical): Don't hard-code the search_path.
+
+-- Uncomment this line if you need to run this query manually.
+--SET search_path TO "MC_results_database";
+
+-- Uncomment this line if you need to redefine this table.
+--DROP TABLE IF EXISTS "sim_annual";
+
+CREATE TABLE IF NOT EXISTS sim_annual
 (
     "TechID" text COLLATE pg_catalog."default",
     "SizingID" text COLLATE pg_catalog."default",
@@ -31,9 +39,12 @@ CREATE TABLE IF NOT EXISTS test_schema.sim_annual
     thm_shw real,
     deskw_ltg real,
     deskw_equ real,
-    lastmod text COLLATE pg_catalog."default"
+    lastmod text COLLATE pg_catalog."default",
+    -- 2025-09-29 Nicholas Fette (Solaris Technical)
+    -- Cause explicit error if user tries to load duplicate data
+    PRIMARY KEY ("TechID", "SizingID", "BldgType", "BldgVint", "BldgLoc", "BldgHVAC", "tstat", "normunit")
 )
 WITH (
     OIDS = FALSE
 )
-TABLESPACE pg_default;
+TABLESPACE pg_default;

scripts/energy savings/commercial/support_and_setup_tables/create_sim_hourly_wb.sql

@@ -1,8 +1,14 @@
 -- Table: MC_results_database.sim_hourly_wb
 
--- DROP TABLE "MC_results_database".sim_hourly_wb;
+-- 2025-09-29 Nicholas Fette (Solaris Technical): Don't hard-code the search_path.
 
-CREATE TABLE IF NOT EXISTS "MC_results_database".sim_hourly_wb
+-- Uncomment this line if you need to run this query manually.
+--SET search_path TO "MC_results_database";
+
+-- Uncomment this line if you need to redefine this table.
+--DROP TABLE IF EXISTS sim_hourly_wb;
+
+CREATE TABLE IF NOT EXISTS sim_hourly_wb
 (
     "TechID" text COLLATE pg_catalog."default",
     "SizingID" text COLLATE pg_catalog."default",
@@ -37,7 +43,10 @@ CREATE TABLE IF NOT EXISTS "MC_results_database".sim_hourly_wb
     hr22 real,
     hr23 real,
     hr24 real,
-    lastmod text COLLATE pg_catalog."default"
+    lastmod text COLLATE pg_catalog."default",
+    -- 2025-09-29 Nicholas Fette (Solaris Technical)
+    -- Cause explicit error if user tries to load duplicate data. This helps avoid accidental mistakes.
+    PRIMARY KEY ("TechID", "SizingID", "BldgType", "BldgVint", "BldgLoc", "BldgHVAC","tstat","enduse","daynum")
 )
 WITH (
     OIDS = FALSE