#!/bin/sh
# Systematically create/update consolidated energy data (typically CSVs).
# In the simple case, the base name of the script to generate a file (.sh)
# is the same as the output file base name (.csv).
# Will attempt to run to the end but return non-zero status in case of error.
# The basename form of each is:
#     std/<variable>/<granularity>/<source>/<variable>-<granularity>-<source>

# Uses some parallelism to reduce wall-clock time.

# Enable performance timestamps with -t flag or debug setting...
PERFTIMESTAMPS=false
if [ "X-t" = "X$1" ]; then
    PERFTIMESTAMPS=true
    shift
elif [ "" != "$DEBUGPGBUILDPERF" ]; then
    PERFTIMESTAMPS=true
fi
if [ "true" = "$PERFTIMESTAMPS" ]; then
    # Default to portable seconds timestamp.
    PERFTIMESTAMPCMD="date +%s"
    # Try for milliseconds if available.
    if [ "$(date +%3N)" != "3N" ]; then
        PERFTIMESTAMPCMD="date +%H:%M:%S.%3N";
    elif [ "$(gdate +%3N)" != "" ] && [ "$(gdate +%3N)" != "3N" ]; then
        PERFTIMESTAMPCMD="gdate +%H:%M:%S.%3N";
    fi
    # Always measure whole script time in seconds...
    PERFTIMESTAMPSTART="$(date +%s)"
fi
"$PERFTIMESTAMPS" && echo "INFO: TS: START: $($PERFTIMESTAMPCMD): $(date -u): $(uname -a)" 1>&2

#echo "INFO: $0 $*" 1>&2
# All executed OK if 0.
status=0

# Flag to touch after run, to allow downstream consumers to trigger.
RUNFLAG=data/.flags/consolidated.flag

# Top directory for the consolidate configuration and outputs.
CONSOLIDDIR=data/consolidated
# Standardised output data directory.
CONSOLIDOUTDIR=data/consolidated/energy/std
# Configuration files.
CONFIGDATASOURCES="$CONSOLIDDIR/config_data_sources.csv"
CONFIGGRANULARITY="$CONSOLIDDIR/config_granularity.csv"
CONFIGVARIABLES="$CONSOLIDDIR/config_variables.csv"
CONFIGSYNTH=$CONSOLIDDIR/config_synthetic.csv

# Location of the standardised generation scripts.
# These generate output on stdout (don't worry about file management).
# These have names in a standard generateable/parseable format.
CONSOLIDSTDSCRDIR=script/consolidate/energy/std

# Output format (on stdout) is:
#date-time,device,coverage,variable,comment
# eg for yearly meter grid imports:
#YYYY,meter,coverage,imp,comment

# Alternatives:
#   * Enumerate all possible data sources etc and find which scripts exist.
#   * Find which scripts exist and match config (and parse the names).
#   * Some combination of the above.
#
# Note that coarser-granularity entries may be run less often.


# Use a single (thread-safe) working output file at the top of the std area.
# Assume that this is the same partition as the destination
# so that an atomic mv works as we would like.
WORKFILETMPDIR="$CONSOLIDOUTDIR/.tmp"
if [ ! -d "$WORKFILETMPDIR" ]; then mkdir "$WORKFILETMPDIR" || exit 1; fi
WORKFILE="$WORKFILETMPDIR/.consolidating.$$.tmp"
rm -f "$WORKFILE"
# Base name for parallel task work files.
PARWORKFILEBASE="$WORKFILETMPDIR/.consolidating.$$.par.tmp."
# Name for parallel error flag file."
PARERRFLAG="$WORKFILETMPDIR/.consolidating.$$.par.err.flag"
rm -f "$PARERRFLAG"
# Trap unexpected exits with tidyup.
trap "/bin/rm -f '$WORKFILE' '$PARERRFLAG' '${PARWORKFILEBASE}*'; exit 1" 1 2 15


# Extract permitted source names.
DATASOURCES="$(awk -F, < "$CONFIGDATASOURCES" '/^[^#]/ {print $1}')"
# Extract permitted granularities.
GRANULARITIES="$(awk -F, < "$CONFIGGRANULARITY" '/^[^#]/ {print $1}')"
# Extract permitted variables (ie reported-on quantities).
VARIABLES="$(awk -F, < "$CONFIGVARIABLES" '/^[^#]/ {print $1}')"


# For parallelising...
# (Given that the target machines have 4 or 8 cores.)
# Number of CPUs.
NPROC="$(nproc)"
# Number of distinct granularities.
NGRAN="$(echo "$GRANULARITIES" | wc -w | awk '{print $1}')"
# Default parallelism.
# One more than the number of granularities to have two same in exec window.
NPAR="$(expr 1 + $NGRAN)"
# If more CPUs than default parallelism, bump it up.
if [ "$NPROC" -gt "$NPAR" ]; then NPAR="$NPROC"; fi
#echo "INFO: NPAR: $NPAR"


# DIRECT GENERATION PER VAR/GRAN/SRC.
# Run direct gen with parallelism; underlying scripts must be safe for this.
# Script of form $CONSOLIDSTDSCRDIR/$VAR/$GRAN/$SRC/$VAR-$GRAN-$SRC.sh
echo "INFO: DIRECT GENERATION of uniform leaf datasets..." 1>&2
"$PERFTIMESTAMPS" && echo "INFO: TS: START DIRECT GEN: $($PERFTIMESTAMPCMD)" 1>&2
# Task list for DIRECT GEN as list of named scripts to run.
TASKLISTDIRECTGEN=
# Enumerate over variables first, seeing what support there is.
for variable in $VARIABLES;
    do
    vardir="$CONSOLIDSTDSCRDIR/$variable"
    if [ ! -d "$vardir" ]; then
        #echo "INFO: no direct support for variable $variable." 1>&2
        continue
    fi
    for granularity in $GRANULARITIES;
        do
        grandir="$vardir/$granularity"
        if [ ! -d "$grandir" ]; then continue; fi
        # Will skip "synth" if present to do all direct generation first.
        for datasource in $DATASOURCES;
            do
            datasourcedir="$grandir/$datasource"
            # Skip 'synth' pseudo-source.
            if [ "synth" = "$datasource" ]; then continue; fi
            # If the data source does not list the variable, sliently skip.
            if [ "" = "$(egrep '^'"$datasource"',' < "$CONFIGDATASOURCES" | egrep ','"$variable"'(,|$)')" ]; then
                continue
            fi
            # Base name of generation script and output name.
            basename="$variable-$granularity-$datasource"
            # Full path name of generation script.
            scriptpath="$datasourcedir/$basename.sh"
            # Skip if no such script.
            if [ ! -s "$scriptpath" ]; then continue; fi
TASKLISTDIRECTGEN="$TASKLISTDIRECTGEN $basename"
            done
        done
    done
echo "$TASKLISTDIRECTGEN" | \
    xargs -r -n 1 -P "$NPAR" \
        sh script/consolidate/energy/directGenWrap.sh "$PARERRFLAG"
"$PERFTIMESTAMPS" && echo "INFO: TS: END DIRECT GEN: $($PERFTIMESTAMPCMD)" 1>&2


# SYNTHETIC GENERATION PER VAR/GRAN/SRC.
# Supports some parallelism.
echo "INFO: SYNTHETIC GENERATION of uniform leaf datasets..." 1>&2
# Task list for SYNTH GEN as list of variable-granularity-datasource triples.
TASKLISTSYNGEN=
"$PERFTIMESTAMPS" && echo "INFO: TS: START SYNTH GEN: $($PERFTIMESTAMPCMD)" 1>&2
# Enumerate over granularity first, to start parallel big tasks early.
for granularity in $GRANULARITIES;
    do
    grandir="$vardir/$granularity"
    for variable in $VARIABLES;
        do
        vardir="$CONSOLIDSTDSCRDIR/$variable"
        if [ "" = "$(egrep < "$CONFIGSYNTH" '^'"$variable"',...')" ]; then
            continue
        fi
        datasource="synth"
        # If the data source does not list the variable, sliently skip.
        if [ "" = "$(egrep '^'"$datasource"',' < "$CONFIGDATASOURCES" | egrep ','"$variable"'(,|$)')" ]; then
            #echo "INFO: no support for variable $variable for data source $datasource" 1>&2
            continue
        fi
        # There is a plausible task to run...
        # Base name of task and output file.
        basename="$variable-$granularity-$datasource"
        TASKLISTSYNGEN="$TASKLISTSYNGEN $basename"
        done
    done
# Have xargs run the tasks in parallel (max NPAR processes)...
# Run each job with exactly one task name.
# Note -t (verbose) option on xargs if needed.
echo "$TASKLISTSYNGEN" | \
    xargs -r -n 1 -P "$NPAR" \
        sh script/consolidate/energy/synthSources.sh -p "$PARERRFLAG"
"$PERFTIMESTAMPS" && echo "INFO: TS: END SYNTH GEN: $($PERFTIMESTAMPCMD)" 1>&2


# COLLAPSE TO COARSER GRANULARITY
# Applies to direct values to fill in gaps.
echo "INFO: COLLAPSING to coarser granularities (finest first: $(echo $GRANULARITIES))..." 1>&2
"$PERFTIMESTAMPS" && echo "INFO: TS: START COLLAPSE: $($PERFTIMESTAMPCMD)" 1>&2
for variable in $VARIABLES;
    do
    vardir="$CONSOLIDSTDSCRDIR/$variable"
    PREVgranularity=""
    for granularity in $GRANULARITIES;
        do
        grandir="$vardir/$granularity"
        for datasource in $DATASOURCES;
            do
            if [ "synth" = "$datasource" ]; then continue; fi
            datasourcedir="$grandir/$datasource"
            # If the data source does not list the variable, sliently skip.
            if [ "" = "$(egrep '^'"$datasource"',' < "$CONFIGDATASOURCES" | egrep ','"$variable"'(,|$)')" ]; then
                continue
            fi
            # Base name of generation script and output name.
            basename="$variable-$granularity-$datasource"
            # Full path name of generation script.
            scriptpath="$datasourcedir/$basename.sh"
            outputdir="$CONSOLIDOUTDIR/$variable/$granularity/$datasource"
            OUTFILE="$outputdir/$basename.csv"
            if [ -s "$scriptpath" ]; then continue; fi
            #echo "INFO: no direct support for variable $variable / granularity $granularity / data source $datasource." 1>&2
            # No support for generating this output directly.
            # Automatically consolidate from finer data if available.
            # Only allow one level of consolidation at a time.
            COLLAPSESCRIPT="script/consolidate/energy/collapse${PREVgranularity}to${granularity}.sh"
            if [ ! -s "${COLLAPSESCRIPT}" ]; then continue; fi
            # Collapse script exists.
#echo "INFO: exists: ${COLLAPSESCRIPT}" 1>&2
            FINERDATA=$CONSOLIDOUTDIR/$variable/${PREVgranularity}/$datasource/$variable-$PREVgranularity-$datasource.csv
            if [ -s "$FINERDATA" ]; then
#echo "INFO: potentially collapsing: $COLLAPSESCRIPT ${FINERDATA}" 1>&2
                if [ ! -s "$OUTFILE" ] || [ "$FINERDATA" -nt "$OUTFILE" ]; then
                    # Finer-grain data exists and is newer...
                    ###
                    # Collapse from finer-grain data.
                    ###
                    if [ ! -d $outputdir ]; then mkdir -p $outputdir; fi
#echo "INFO: collapsing: $COLLAPSESCRIPT ${FINERDATA}" 1>&2
                    if sh "$COLLAPSESCRIPT" "$FINERDATA" > "$WORKFILE"; then
                        # Atomically move into place.
                        sh script/replacePublishedFile.sh "$WORKFILE" "$OUTFILE"
                        #echo "INFO: collapsed $FINERDATA to $OUTFILE" 1>&2
                    else
                        # Failed: remove any broken/partial result.
                        rm -f "$WORKFILE"
                        status=1
                        echo "ERROR: failed: $COLLAPSESCRIPT $FINERDATA." 1>&2
                    fi
                fi
            fi
            done
        PREVgranularity="$granularity"
        done
    done
"$PERFTIMESTAMPS" && echo "INFO: TS: END COLLAPSE: $($PERFTIMESTAMPCMD)" 1>&2


# CONSOLIDATE ACROSS DATA SOURCES
# Applies to direct and synthetic values.
echo "INFO: CONSOLIDATING..." 1>&2
"$PERFTIMESTAMPS" && echo "INFO: TS: START CONSOLIDATE: $($PERFTIMESTAMPCMD)" 1>&2
# Task list for consolidation.
#TASKLISTCONS=
# Fine granularity slow tasks first.
for granularity in $GRANULARITIES;
    do
    grandir="$vardir/$granularity"
    for variable in $VARIABLES;
        do
        vardir="$CONSOLIDSTDSCRDIR/$variable"
        # Source leaf time series to be consolidated into this putative result.
        GENERATEDATLEAF=""
        LEAFSOURCES=""
        uptodate=true;
        OUTFILE="$CONSOLIDOUTDIR/$variable/$granularity/$variable-$granularity.csv"
        if [ ! -s "$OUTFILE" ]; then uptodate=false; fi
        for datasource in $DATASOURCES;
            do
            datasourcedir="$grandir/$datasource"
            # If the data source does not list the variable, sliently skip.
            if [ "" = "$(egrep '^'"$datasource"',' < "$CONFIGDATASOURCES" | egrep ','"$variable"'(,|$)')" ]; then
                continue
            fi
            # Base name of input name.
            basename="$variable-$granularity-$datasource"
            inputdir="$CONSOLIDOUTDIR/$variable/$granularity/$datasource"
            INFILE="$inputdir/$basename.csv"
            if [ -s "$INFILE" ]; then
                GENERATEDATLEAF="$INFILE $GENERATEDATLEAF";
                LEAFSOURCES="$LEAFSOURCES,$datasource"
                if [ "$INFILE" -nt "$OUTFILE" ]; then uptodate=false; fi
            fi
            done
        # If no suitable leaf sources, skip.
        if [ "" = "$GENERATEDATLEAF" ]; then continue; fi
        # If does not need computing (inputs are older), skip.
        if "$uptodate"; then continue; fi
        ###
        # Consolidate across data sources for this variable and granularity.
        ###
        #TASKLISTCONS="$TASKLISTCONS $variable-$granularity-$LEAFSOURCES"
        #echo "INFO: consolidating for v:$variable g:$granularity: $GENERATEDATLEAF..." 1>&2
        rm -f "$WORKFILE"
        if sh script/consolidate/energy/mergeAcrossSources.sh $GENERATEDATLEAF > "$WORKFILE"; then
            # Atomically move into place.
            sh script/replacePublishedFile.sh "$WORKFILE" "$OUTFILE"
        else
            echo "ERROR: consolidation failed for $variable-$granularity." 1>&2
            status=1
        fi
        done
    done
#echo "INFO: TASKLISTCONS: $TASKLISTCONS" 1>&2
"$PERFTIMESTAMPS" && echo "INFO: TS: END CONSOLIDATE: $($PERFTIMESTAMPCMD)" 1>&2


# GENERATE DERIVED NON-CSV OUTPUTS (sonfication)
echo "INFO: SONIFYING..." 1>&2
"$PERFTIMESTAMPS" && echo "INFO: TS: START SONIFY: $($PERFTIMESTAMPCMD)" 1>&2
if sh script/sonifyConsolidatedEnergyOutputs.sh > /dev/null; then
    echo "INFO: sonfication done." 1>&2
else
    echo "ERROR: sonfication failed." 1>&2
    status=1
fi
"$PERFTIMESTAMPS" && echo "INFO: TS: END SONIFY: $($PERFTIMESTAMPCMD)" 1>&2


"$PERFTIMESTAMPS" && echo "INFO: TS: END: $($PERFTIMESTAMPCMD)" 1>&2
if [ "true" = "$PERFTIMESTAMPS" ]; then
    # Measure whole script time in seconds...
    PERFTIMESTAMPEND="$(date +%s)"
    echo "INFO: TS: TOTAL: $(expr "$PERFTIMESTAMPEND" - "$PERFTIMESTAMPSTART")s" 1>&2
fi

# Final catch-all for parallel task failures.
if [ -e "$PARERRFLAG" ]; then
    echo "ERROR: a parallel task failed..." 1>&2
    status=1
fi

# Tidy up.
rm -f "${WORKFILE}" "${PARERRFLAG}"
# Indicate if files may have been successfully updated.
if [ 0 = "$status" ]; then touch $RUNFLAG; else echo "ERROR: $status" 1>&2; fi
exit $status


# Performance Notes

# DHD20260117: SYNTH GEN (~36s) seems to be a good place to focus.
#% sh script/updateConsolidatedEnergyOutputs.sh | & tee /tmp/log
#% egrep '^INFO: TS:' /tmp/log
#INFO: TS: START: 04.980: Sat 17 Jan 2026 15:08:04 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 05.004
#INFO: TS: END DIRECT GEN: 15.427
#INFO: TS: START SYNTH GEN: 15.439
#INFO: TS: END SYNTH GEN: 51.315
#INFO: TS: START COLLAPSE: 51.327
#INFO: TS: END COLLAPSE: 53.507
#INFO: TS: START CONSOLIDATE: 53.519
#INFO: TS: END CONSOLIDATE: 58.544
#INFO: TS: END: 00.610
#INFO: TS: TOTAL: 116s
#
# in lowpowermode (SYNTH GEN now ~54s):
#INFO: TS: START: 45.917: Sat 17 Jan 2026 15:50:45 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 45.958
#INFO: TS: END DIRECT GEN: 02.006
#INFO: TS: START SYNTH GEN: 02.024
#INFO: TS: END SYNTH GEN: 56.216
#INFO: TS: START COLLAPSE: 56.235
#INFO: TS: END COLLAPSE: 59.476
#INFO: TS: START CONSOLIDATE: 59.494
#INFO: TS: END CONSOLIDATE: 42.224
#INFO: TS: END: 45.309
#INFO: TS: TOTAL: 180s
#
# after first parallelism attempt in SYNTH GEN, not lowpowermode, 3s saved
# (and capturing sonification timing):
#INFO: TS: START: 25.666: Sat 17 Jan 2026 17:05:25 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 25.693
#INFO: TS: END DIRECT GEN: 36.062
#INFO: TS: START SYNTH GEN: 36.074
#INFO: TS: END SYNTH GEN: 08.906
#INFO: TS: START COLLAPSE: 08.919
#INFO: TS: END COLLAPSE: 11.098
#INFO: TS: START CONSOLIDATE: 11.110
#INFO: TS: END CONSOLIDATE: 15.555
#INFO: TS: START SONIFY: 15.567
#INFO: TS: END SONIFY: 17.443
#INFO: TS: END: 17.455
#INFO: TS: TOTAL: 112s

# DHD20260118: rewritten SYNTH parallelism with xargs: down from ~36s to ~13s.
#INFO: TS: START: 53.311: Sun 18 Jan 2026 17:56:53 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 53.357
#INFO: TS: END DIRECT GEN: 03.876
#INFO: TS: START SYNTH GEN: 03.890
#INFO: TS: END SYNTH GEN: 16.485
#INFO: TS: START COLLAPSE: 16.498
#INFO: TS: END COLLAPSE: 18.254
#INFO: TS: START CONSOLIDATE: 18.266
#INFO: TS: END CONSOLIDATE: 22.657
#INFO: TS: START SONIFY: 22.669
#INFO: TS: END SONIFY: 24.565
#INFO: TS: END: 24.577
#INFO: TS: TOTAL: 91s

# DHD20260119: added minutes to timestamp to make CONSOLIDATE ~64s clearer.
#INFO: TS: START: 35:04.503: Mon 19 Jan 2026 10:35:04 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 35:04.548
#INFO: TS: END DIRECT GEN: 35:15.026
#INFO: TS: START SYNTH GEN: 35:15.038
#INFO: TS: END SYNTH GEN: 35:27.504
#INFO: TS: START COLLAPSE: 35:27.516
#INFO: TS: END COLLAPSE: 35:29.275
#INFO: TS: START CONSOLIDATE: 35:29.287
#INFO: TS: END CONSOLIDATE: 36:33.784
#INFO: TS: START SONIFY: 36:33.796
#INFO: TS: END SONIFY: 36:35.763
#INFO: TS: END: 36:35.775
#INFO: TS: TOTAL: 91s
#
# Reduced algo order to O(n) from O(n^2) in merge script.
#INFO: TS: START DIRECT GEN: 16:43:50.399
#INFO: TS: END DIRECT GEN: 16:44:00.816
#INFO: TS: START SYNTH GEN: 16:44:00.828
#INFO: TS: END SYNTH GEN: 16:44:13.308
#INFO: TS: START COLLAPSE: 16:44:13.320
#INFO: TS: END COLLAPSE: 16:44:15.070
#INFO: TS: START CONSOLIDATE: 16:44:15.084
#INFO: TS: END CONSOLIDATE: 16:44:17.626
#INFO: TS: START SONIFY: 16:44:17.638
#INFO: TS: END SONIFY: 16:44:19.458
#INFO: TS: END: 16:44:19.470
#INFO: TS: TOTAL: 29s
#
# Reduced SYNTH GEN script order to O(n).
#INFO: TS: START: 20:28:21.447: Mon 19 Jan 2026 20:28:21 UTC: Darwin localhost 25.2.0 Darwin Kernel Version 25.2.0: Tue Nov 18 21:09:55 PST 2025; root:xnu-12377.61.12~1/RELEASE_ARM64_T8103 arm64
#INFO: TS: START DIRECT GEN: 20:28:21.503
#INFO: TS: END DIRECT GEN: 20:28:31.911
#INFO: TS: START SYNTH GEN: 20:28:31.923
#INFO: TS: END SYNTH GEN: 20:28:32.403
#INFO: TS: START COLLAPSE: 20:28:32.415
#INFO: TS: END COLLAPSE: 20:28:34.159
#INFO: TS: START CONSOLIDATE: 20:28:34.171
#INFO: TS: END CONSOLIDATE: 20:28:36.827
#INFO: TS: START SONIFY: 20:28:36.840
#INFO: TS: END SONIFY: 20:28:38.810
#INFO: TS: END: 20:28:38.822
#INFO: TS: TOTAL: 17s

# DHD20260220: parallelised initial direct data extraction step for lolz.
#INFO: TS: START: 17:01:24.709: Fri 20 Feb 2026 17:01:24 UTC: Darwin localhost 25.3.0 Darwin Kernel Version 25.3.0: Wed Jan 28 20:53:01 PST 2026; root:xnu-12377.81.4~5/RELEASE_ARM64_T8103 arm64
#INFO: DIRECT GENERATION of uniform leaf datasets...
#INFO: TS: START DIRECT GEN: 17:01:24.755
#INFO: TS: END DIRECT GEN: 17:01:28.177
#INFO: SYNTHETIC GENERATION of uniform leaf datasets...
#INFO: TS: START SYNTH GEN: 17:01:28.189
#INFO: TS: END SYNTH GEN: 17:01:28.658
#INFO: COLLAPSING to coarser granularities...
#INFO: TS: START COLLAPSE: 17:01:28.671
#INFO: TS: END COLLAPSE: 17:01:30.459
#INFO: CONSOLIDATING...
#INFO: TS: START CONSOLIDATE: 17:01:30.471
#INFO: TS: END CONSOLIDATE: 17:01:33.086
#INFO: SONIFYING...
#INFO: TS: START SONIFY: 17:01:33.099
#INFO: sonified!
#INFO: all std energy MIDI files, flattened: -rw-r--r--  1 dhd  staff  319063 20 Feb 17:01 data/consolidated/energy/col/stdMIDI.zip
#INFO: monthly house std energy MIDI files, flattened: -rw-r--r--  1 dhd  staff  33414 20 Feb 17:01 data/consolidated/energy/col/stdMIDI-house-M.zip
#INFO: yearly house std energy MIDI files, flattened: -rw-r--r--  1 dhd  staff  25989 20 Feb 17:01 data/consolidated/energy/col/stdMIDI-house-Y.zip
#INFO: sonfication done.
#INFO: TS: END SONIFY: 17:01:34.841
#INFO: TS: END: 17:01:34.853
#INFO: TS: TOTAL: 10s