#!/bin/bash
################################################################################
# sdcto123 imports E-Total-values from SunnyDataControl *.suo (SMA-Sunny Online)
# files into 123Solar energy20YY.csv files
#
# Version History:
# v0.1   2014-04-25   initial release
#
# Project Homepage: datacrumbs.de/123solar
#
# Contact: mario@datacrumbs.de
#
# Copyright (C) 2014 Mario Hirt
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# See <http://www.gnu.org/licenses/> for the full License description.
################################################################################


function showhelp() {
  echo "usage: sdcto123 [-dxyz] [-r E-Total-RowNumber] [-s MMDD] [-t E-Total-InitV] SunnyDataControlDir outDir"
  echo "              [-d] : enable Debugmode / show verbose output"
  echo "              [-x] : Debugoption: disable first stage"
  echo "              [-y] : Debugoption: disable second stage"
  echo "              [-z] : Debugoption: disable third stage"
  echo "              [-r RowNumber] : set RowNumber for E-Total values in .suo files"
  echo "              [-s MMDD] : fill up the first year beginning at date MMDD"
  echo "              [-t X.X] : initial powermeter value (float) at given date MMDD"
  echo "              SunnyDataControlDir : home of your YYMMDDXX.suo csv-style files"
  echo "              outDir : temp- and outputdir for 123Solar energy20YY.csv files"
  exit 2
}

function cleanup() {
  if [ -n "$outDir" ] && [ -d $outDir ]; then
    rm $outDir*.tmp > /dev/null 2>&1
    [ -n "$dbg" ] && echo -e "\nRemove *.cs1 temp files in $outDir before next run or choose different outDir or use -x option!" || rm $outDir*.cs1 > /dev/null 2>&1
    [ "$(ls -A $outDir)" ] || rmdir $outDir
  fi
}
trap cleanup EXIT

function csvupdate() {
  if [ -f $1 ]; then
    txtFnd=$2
    txtRpl=$2,$3,$4
    sed -i -e "s/${txtFnd}.*/${txtRpl}/" $1
  fi
}

function csvwrite() {
  [ -f $1 ] || touch $1
  if [ -z $4 ]; then
    echo "$2,$3" >> $1
  else
    echo "$2,$3,$4" >> $1
  fi
}


####################
# Input handling

dbg=""
eTotRowNrOverride=""
firstStageOverride=""
secondStageOverride=""
thirdStageOverride=""
firstYStartD=""
firstYeTotInitV=""

OPTINDBckp=$OPTIND
OPTARGBckp=$OPTARG
OPTERRBckp=$OPTERR

optSpec=":hdxyzr:s:t:-:"
while getopts "$optSpec" optChar; do
  case "${optChar}" in
    -)
      case "${OPTARG}" in
        help)
          showhelp
          ;;
        *)
          if [ "$OPTERR" = 1 ] && [ "${optSpec:0:1}" = ":" ]; then
            echo "Invalid option: --${OPTARG}" >&2
            showhelp
          fi;;
      esac;;
    d)
      echo "Option -d: Debugmode ENABLED"
      dbg="true"
      ;;
    x)
      echo "Option -x: 1st Stage DISABLED"
      firstStageOverride="true"
      ;;
    y)
      echo "Option -y: 2nd Stage DISABLED"
      secondStageOverride="true"
      ;;
    z)
      echo "Option -z: 3rd Stage DISABLED"
      thirdStageOverride="true"
      ;;
    r)
      echo -n "Option -r: "
      if [ ! -z "${OPTARG##*[!0-9]*}" ]; then
        echo "E-Total-RowNumber set to $OPTARG"
        eTotRowNr="$OPTARG"
	eTotRowNrOverride="true"
      else
        echo "has to be an unsigned integer!"
        showhelp
      fi;;
    s)
      echo -n "Option -s: "
      if [[ $OPTARG = [0-1][0-9][0-3][0-9] ]]; then
	echo "Inverter went productive at $OPTARG (MMDD)"
	firstYStartD=$OPTARG
      else
        echo "doesn't even look like a valid date!"
        showhelp
      fi;;
    t)
      echo -n "Option -t: "
      if [[ $OPTARG =~ ^[0-9]+\.+[0-9]+$ ]]; then
	echo "Initial powermeter value was $OPTARG"
        firstYeTotInitV=$OPTARG
      else
        echo "has to be an unsigned float of the form X.X!"
	showhelp
      fi;;
    h)
      showhelp
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      showhelp
      ;;
    *)
      if [ "$OPTERR" != 1 ] || [ "${optSpec:0:1}" = ":" ]; then
        echo "Non-option argument: '-${OPTARG}'" >&2
        showhelp
      fi;;
  esac
done

shift $(($OPTIND - 1))

if [ -z "$1" ] || [ -z "$2" ]; then
  showhelp
fi

sdcDir=$1
[ "${sdcDir: -1}" = "/" ] || sdcDir=$sdcDir/
[ ! -d $sdcDir ] && echo "ERROR: SunnyDataControlDir not found." >&2 && exit 5
[ ! "$(ls -A $sdcDir | grep -i '^[0-9]\{8\}.suo$')" ] && echo "ERROR: No *.suo files in SunnyDataControlDir." >&2 && exit 5

outDir=$2
[ "${outDir: -1}" = "/" ] || outDir=$outDir/
[ -d $outDir ] || mkdir -p $outDir

OPTIND=$OPTINDBckp
OPTARG=$OPTARGBckp
OPTERR=$OPTERRBckp

[ -n "$dbg" ] && echo -e "InputValues: E-Total-RowNumber: $eTotRowNr, $eTotRowNrOverride, firstYStartD: $firstYStartD,\n  SunnyDataControlDir: $sdcDir, outDir: $outDir,\n  StageOverrides: $firstStageOverride, $secondStageOverride, $thirdStageOverride"


if [ -z "$firstStageOverride" ]; then

####################
# Generate temporary datafiles from SunnyDataControl .suo files
echo "(1st Stage) Reading SunnyDataControl csv files from $sdcDir"
for f in $(ls $sdcDir | grep .suo); do
  fDate=20${f:0:6}
  outFile=energy20${f:0:2}.cs1
  [ -f "$outDir$outFile" ] || touch $outDir$outFile

  # Autodetect E-Total-RowNumber
  if [ -z "$eTotRowNrOverride" ]; then
    eTotRowNr=""
    fDescrLine=$(sed -n '3{p;q}' $sdcDir$f)
    fDescrArr=(${fDescrLine//';'/ })
    c=0
    for i in "${fDescrArr[@]}"; do
      if [ "$i" = "E-Total" ]; then
        eTotRowNr="$c"
        break
      fi
      c=$((c + 1))
    done
    [ -z "$eTotRowNr" ] && echo "$f: ERROR: Autodetection of E-Total-RowNumber failed!" >&2 && continue
  fi

  # Read first and last data line in file, extract E-Total values
  eTotStartV=$(awk -F ";" -vrn=${eTotRowNr} 'NR > 5 { exit }; NR == 5 { print $rn }' $sdcDir$f)
  eTotEndV=$(tail -1 $sdcDir$f | awk -F ";" -vrn=${eTotRowNr} '{ print $rn }')

  # Check, whether first and last data line entries are valid
  if [ -n "$eTotStartV" ] && [[ $eTotStartV =~ ^[0-9]+\.+[0-9]+$ ]] && [ -n "$eTotEndV" ] && [[ $eTotEndV =~ ^[0-9]+\.+[0-9]+$ ]]; then
    # If working on first file of a day -> write out data
    if [ "${f:6:2}" = "00" ]; then
      [ -n "$dbg" ] && echo "$f: $outFile, $eTotStartV, $eTotEndV"
      csvwrite $outDir$outFile $fDate $eTotStartV $eTotEndV
    else
      # Working on a file which has a predecessor on the same day -> update data
      if [ -f $sdcDir${f:0:6}00${f:8} ]; then
        eTotStartVFirst=$(awk -F ";" -vrn=${eTotRowNr} 'NR > 5 { exit }; NR == 5 { print $rn }' $sdcDir${f:0:6}00${f:8})
        if [ -n "$eTotStartVFirst" ] && [[ $eTotStartVFirst =~ ^[0-9]+\.+[0-9]+$ ]]; then
          [ -n "$dbg" ] && echo "$f: Updated values (${f:0:6}00${f:8}): $eTotStartVFirst, $eTotEndV"
	  csvupdate $outDir$outFile $fDate $eTotStartVFirst $eTotEndV
        else
          # First file of day maleformed -> search for existing data in outFile -> update or write incomplete data
          eTotStartVFirst=$(awk -F "," -vlnp=${fDate} '$0 ~ "^"lnp { print $2 }' $outDir$outFile)
          if [ -n "$eTotStartVFirst" ]; then
            [ -n "$dbg" ] && echo "$f: File ${f:0:6}00${f:8} maleformed, updated values: $eTotStartVFirst, $eTotEndV"
	    csvupdate $outDir$outFile $fDate $eTotStartVFirst $eTotEndV
	  else
            echo "$f: WARNING: File ${f:0:6}00${f:8} maleformed, incomplete values: $eTotStartV, $eTotEndV"
            csvwrite $outDir$outFile $fDate $eTotStartV $eTotEndV
          fi
        fi
      else
          # First file of day missing -> search for existing data in outFile -> update or write incomplete data 
          eTotStartVFirst=$(awk -F "," -vlnp=${fDate} '$0 ~ "^"lnp { print $2 }' $outDir$outFile)
          if [ -n "$eTotStartVFirst" ]; then
            [ -n "$dbg" ] && echo "$f: File (${f:0:6}00${f:8}) missing, updated values: $eTotStartVFirst, $eTotEndV"
	    csvupdate $outDir$outFile $fDate $eTotStartVFirst $eTotEndV
	  else
            echo "$f: WARNING: File ${f:0:6}00${f:8} missing, incomplete values: $eTotStartV, $eTotEndV"
            csvwrite $outDir$outFile $fDate $eTotStartV $eTotEndV
          fi
      fi
    fi
  else
      echo "$f: ERROR: File maleformed." >&2
  fi
done

fi
if [ -z "$secondStageOverride" ] || [ -z "$thirdStageOverride" ]; then

####################
# Transform temporary datafiles into energy20YY.csv output files
for f in $(ls $outDir | grep .cs1); do
  echo -ne "\nWorking on $f:"

  if [ -z "$secondStageOverride" ]; then

  # Eliminate duplicates, sorting entries based on date: only the first entry
  #   for a particular date is preserved. Older values from former runs
  #   are thus NOT overwritten. Instead restart with a clean outDir!
  [ -z "$dbg" ] && echo -n " (2nd Stage) sorting"
  sort -u -k1.1,1.8 $outDir$f > $outDir$f.tmp
  mv $outDir$f.tmp $outDir$f

  yyyy=${f:6:4}
  outFile=energy$yyyy.csv
  yIndFile=$yyyy.tmp
  fBefore=energy$((yyyy - 1)).cs1
  fNext=energy$((yyyy + 1)).cs1

  # Generate year index
  if [ ! -f $outDir$yIndFile ]; then
    touch $outDir$yIndFile
    for m in {1..12}; do
      for d in $(cal $m $yyyy | grep "^ *[0-9]"); do
        mm=$(printf "%02d" $m)
        dd=$(printf "%02d" $d)
        echo "$yyyy$mm$dd" >> $outDir$yIndFile
      done
    done
  fi

  # Try to match every date from year index with line in temporary datafile
  #   in case of mismatch, fill gaps with synthetic data
  [ -z "$dbg" ] && echo -n ", filling gaps"
  c=0
  cLastMatch=0
  firstYOffset=0
  for l in $(cat $outDir$yIndFile); do
    c=$((c + 1))
    # searching for yearindex l in f
    fMatchLNr=$(awk -vlnp=${l} '$0 ~ "^"lnp { print NR }' $outDir$f)
    if [ -n "$fMatchLNr" ]; then
      [ -n "$dbg" ] && echo -ne "\n$outDir$yIndFile: $c, $l :: $outDir$f: $fMatchLNr"
      if [ "$fMatchLNr" = "$c" ]; then
        [ -n "$dbg" ] && echo -n "  OK!"
        cLastMatch=$c
      else
        [ -n "$dbg" ] && echo -n "  MISSMATCH!"
	if [ "$fMatchLNr" -gt "$c" ]; then
	  echo -ne "\nERROR: More Datalines than days in year $yyyy? $f missordered!" >&2
	  exit 5
        else
          if [ "$c" = "$((cLastMatch + 1))" ]; then
            [ -n "$dbg" ] && echo -ne "\nThe date before was found, no gap to fill!"
            cLastMatch=$c
          else
            eTotStartVMatch=$(awk -F "," -vlnr=${fMatchLNr} 'NR == lnr { print $2 }' $outDir$f)
            # it is the first entry of this year
            if [ "$cLastMatch" -eq "0" ]; then
              if [ -f $outDir$fBefore ]; then
                eTotEndVLastMatch=$(tail -1 $outDir$fBefore | awk -F "," '{ print $3 }')
              else
		if [ -z "$firstYStartD" ]; then
	          [ -n "$dbg" ] && echo -ne "\nFirst year, not filling up!"
                  cLastMatch=$c
                  firstYOffset=$((c - 1))
                  continue
                else
	          startDate=$yyyy$firstYStartD
                  startDateLNr=$(awk -vlnp=${startDate} '$0 ~ "^"lnp { print NR }' $outDir$yIndFile)
                  if [ -z "$startDateLNr" ]; then
                    echo -ne "\nERROR: Given date (-s MMDD) not valid for first found year $yyyy!" >&2
		    exit 5
                  else
                    [ -n "$dbg" ] && echo -ne "\nFilling up from $startDate to $l" >&2
                    if [ "$startDateLNr" -ge "$c" ]; then
                      echo -ne "\nERROR: Set date (-s MMDD) before ${l:4:4} to be valid for first found year $yyyy!" >&2
		      exit 5
                    else
                      cLastMatch=$((startDateLNr - 1))
                      firstYOffset=$((cLastMatch))
                      if [ -z "$firstYeTotInitV" ]; then
                        eTotEndVLastMatch=0
                      else
                        if [ "$(echo $eTotStartVMatch '>' $firstYeTotInitV | bc)" -eq "1" ]; then
                          eTotEndVLastMatch=$firstYeTotInitV
                        else
                          echo -ne "\nERROR: Given initial powermeter value higher than first found E-Total value!" >&2
                          exit 5
                        fi
                      fi
                    fi
                  fi
                fi
              fi
            else
              eTotEndVLastMatch=$(awk -F "," -vlnr=$((${cLastMatch} - ${firstYOffset})) 'NR == lnr { print $3 }' $outDir$f)
            fi
            [ -z "$eTotStartVMatch" ] && eTotStartVMatch=0 && echo -ne "\nWARNING: $l, eTotStartVMatch got no value"
            [ -z "$eTotEndVLastMatch" ] && eTotEndVLastMatch=0 && echo -ne "\nWARNING: $l, eTotEndVLastMatch got no value"
            eTotDiff=$(echo "scale=3;((${eTotStartVMatch} - ${eTotEndVLastMatch}) / (${c} - ${cLastMatch} - 1))" | bc)
            cDiffLastMatch=$((c - (c - cLastMatch) + 1))
            [ -n "$dbg" ] && echo -ne "\ncLastMatch:$cLastMatch, cDiffLastMatch:$cDiffLastMatch, firstYOffset:$firstYOffset, eTotEndVLastMatch:$eTotEndVLastMatch, eTotStartVMatch:$eTotStartVMatch, eTotDiff:$eTotDiff"
            for ((i=$cDiffLastMatch; i<$c; i++)); do
              eTotDate=$(awk "NR==$i" "$outDir$yIndFile")
              eTotStartV=$(echo "scale=3;(${eTotEndVLastMatch} + ((${i} - ${cDiffLastMatch}) * ${eTotDiff}))" | bc)
              eTotEndV=$(echo $eTotStartV + $eTotDiff | bc)
              [ -n "$dbg" ] && echo -ne "\ninsert: $eTotDate, $eTotStartV, $eTotEndV"
              sed -i -e "/${l}/i${eTotDate},${eTotStartV},${eTotEndV}" ${outDir}${f}
            done
            cLastMatch=$c
          fi
	fi
      fi
    fi
  done

  fi
  if [ -z "$thirdStageOverride" ]; then

  # Compare energy 'production' begin and end values for every day,
  #   in case of mismatch, rise end value of former day
  [ -z "$dbg" ] && echo -n ", (3rd Stage) correcting and writing final values"
  c=0
  for l in $(cat $outDir$f); do
    c=$((c + 1))
    eTotArr=(${l//','/ })
    eTotDate=${eTotArr[0]}
    eTotStartV=${eTotArr[1]}
    eTotEndV=${eTotArr[2]}
    eTotStartVNext=$(awk -F "," -vlnr=$((${c} + 1)) 'NR > lnr { exit }; NR == lnr { print $2 }' $outDir$f)
    [ -z "$eTotStartVNext" ] && [ -f $outDir$fNext ] && eTotStartVNext=$(awk -F "," 'NR > 1 { exit }; { print $2 }' $outDir$fNext)
    if [ -n "$eTotStartVNext" ] && [ "$(echo $eTotStartVNext '>' $eTotEndV | bc)" -eq "1" ]; then
      [ -n "$dbg" ] && echo -ne "\n$f:$c:$eTotDate Replacing $eTotEndV with $eTotStartVNext"
      eTotEndV=$eTotStartVNext
      sed -i -e "${c}s/^.*$/${eTotDate},${eTotStartV},${eTotEndV}/" ${outDir}${f}
    fi 

    # Calculate eTotDiff value and write 123Solar energy file
    eTotDiff=$(echo $eTotEndV - $eTotStartV | bc)
    [ "${eTotDiff:0:1}" = "." ] && eTotDiff=0$eTotDiff
    csvwrite $outDir$outFile $eTotDate $eTotDiff
  done

  fi
  echo -n "."
done

fi
echo -e "\ndone."