Raspberry Pi Traffic Monitoring: Difference between revisions
Jump to navigation
Jump to search
Line 441: | Line 441: | ||
# EOF | # EOF | ||
== trafficTruncateLogs.php == | |||
<?php | |||
/* | |||
Script: trafficTruncateLogs.php | |||
Purpose: create truncated logs for trafficPlot.sh, replacing that code in trafficPlot.sh (which is very slow) | |||
Origin: Pi3 | |||
/* Globals */ | |||
$gLogsListPath=$argv[1]; | |||
$gGTimeLowerWatermarkMinutes=$argv[2]; | |||
$gMinutesToKeep=$argv[3]; | |||
$gLogsListPathShortened=$argv[4]; | |||
#print_r($argv); | |||
/* Main */ | |||
/* Read the Original Log-List-File */ | |||
$logListString=file_get_contents($gLogsListPath); | |||
$logListArray=explode("\n", $logListString); | |||
/* Walk thru lines */ | |||
foreach ($logListArray as $logListOnePath) | |||
{ | |||
if (strlen($logListOnePath)>5) | |||
{ | |||
echo ("Working on $logListOnePath...\n"); | |||
/* Create shortened copy of the logfile */ | |||
$logListOnePathShortened=$logListOnePath . ".tmp"; | |||
$cmd="tail -" . ($gMinutesToKeep + 5) . " \"$logListOnePath\" > \"$logListOnePathShortened\""; | |||
#echo ("cmd: $cmd\n"); | |||
system ($cmd); /* LOL: https://stackoverflow.com/questions/15025875/what-is-the-best-way-in-php-to-read-last-lines-from-a-file/15025877 */ | |||
/* Now Walk thru the shortened one, shorten it even more, and check whether it contains only 0 or not */ | |||
$oneLogString=file_get_contents($logListOnePathShortened); | |||
$oneLogArray=explode("\n", $oneLogString); | |||
$oneLogArrayNew=array(); | |||
$foundNonZero=false; | |||
array_unshift($oneLogArray, "TIME\tSENTKB\tRECEIVEDKB"); # gnuplot seeams to skip the first line, so make it at least useful | |||
foreach ($oneLogArray as $oneLogLine) | |||
{ | |||
if (strpos($oneLogLine, "SENT") === false && strlen($oneLogLine) > 5 ) // TBD: Match Line /beginning/ with # | |||
{ | |||
$oneLogLineArray=explode("\t", $oneLogLine); | |||
$time=$oneLogLineArray[0]; | |||
$up=$oneLogLineArray[1]; | |||
$down=$oneLogLineArray[2]; | |||
/* Skip all lines befor given date */ | |||
//awk -F: '{ print ($1 * 60) + ($2) | |||
$timeArray=explode(":", $time); | |||
#print_r($timeArray); | |||
$timeInMinutes=$timeArray[0] * 60 + $timeArray[1]; | |||
if ($timeInMinutes >= $gGTimeLowerWatermarkMinutes) | |||
{ | |||
/* Check for all-zero-file */ | |||
if ($up != 0 || $down != 0) | |||
{ | |||
echo ("\"$up\" and \"$down\", nonzero\n"); | |||
$foundNonZero=true; | |||
} | |||
/* Add line */ | |||
array_push($oneLogArrayNew, $oneLogLine); | |||
} | |||
} // if comment | |||
else | |||
{ | |||
array_push($oneLogArrayNew, $oneLogLine); | |||
} | |||
} // for each shortened log file line | |||
/* Write Array back to file, or even kill the file */ | |||
if ($foundNonZero == true) | |||
{ | |||
echo("Overwriting shortened log with:\n"); | |||
print_r($oneLogArrayNew); | |||
/* Overwrite with (even more) shortened log */ | |||
file_put_contents($logListOnePathShortened, implode("\n", $oneLogArrayNew)); | |||
/* Remind it for later */ | |||
echo("Adding \"$logListOnePathShortened\" to \"$gLogsListPathShortened\"\n"); | |||
file_put_contents($gLogsListPathShortened, $logListOnePathShortened . "\n", FILE_APPEND); | |||
} | |||
else | |||
{ | |||
unlink($logListOnePathShortened); | |||
} | |||
} // plausi logListOnePath | |||
} // foreach original log list | |||
?> | |||
== And this is what you get... == | == And this is what you get... == | ||
[[Image:2019-02-19_23-28.gif]] | [[Image:2019-02-19_23-28.gif]] | ||
= hogwatch = | = hogwatch = |
Revision as of 14:16, 21 February 2019
The Why Of Fry
After I did Raspberry as Pi Ethernet-WiFi-Bridge, I thought it might be nice to monitor traffic via my bridge.
iftop
Installtion
apt-get install iftop
Additional scripts
cronjob
Since only root can access devices traffic, you need to sudo this, or run it in root's crontab:
# Log traffic and generate gnuplots */1 * * * * /bin/bash /home/pi/bin/trafficCheck.sh >> /dev/null 2>&1
trafficCheck.sh
#!/bin/bash set -x # File: trafficCheck.sh # Purpose: Measure traffic and log it. Must be run as root. # Origin: pi3 # # Globals # GBASENAME="trafficCheck" # TBD: get this from the env GSAMPLETIME=40 # in seconds. Same as 3rd average column. DO NOT CHANGE! GFOLDERTMP="/home/pi/var/trafficCheck/tmp" GFOLDERLOGS="/home/pi/var/trafficCheck/logs" GFOLDERWWWTXT="/var/www/html/traffic/archive-txt" GFILEOUTPUT="${GFOLDERTMP}/trafficCheck.out" GDATE="`date +%Y-%m-%d`" GTIME="`date +%H:%M`" GTIMEM="`date +%H-%M`" #GFILELOG="${GFOLDERTMP}/${GDATE}.txt" # Preparations for first run #mkdir -p "${GFOLDERTMP}" #mkdir -p "${GFOLDERLOGS}" # # Functions # function makeKBytes() { # First make bytes... echo "${1}" | grep MB > /dev/null 2>&1 if [ "$?" -eq 0 ]; then # Megabreit! NUMBER="`echo \"${1}\" | cut -dM -f1`" RESULT="`echo \"${NUMBER} * 1024 * 1024\" | bc`" else echo "${1}" | grep KB > /dev/null 2>&1 if [ "$?" -eq 0 ]; then # Kilobyte NUMBER="`echo \"${1}\" | cut -dK -f1`" RESULT="`echo \"${NUMBER} * 1024\" | bc`" else # Bytes NUMBER="`echo \"${1}\" | cut -dB -f1`" RESULT=$NUMBER fi fi #echo RESULT # Make KBytes out of the bytes RESULT="`echo \"${RESULT} / 1024\" | bc`" echo $RESULT } # Sandbox #makeKBytes 2048B #makeKBytes 2048KB #makeKBytes 2048MB #exit 0 # # Main # echo "Init done. `date`" # just for profiling # Use iftop to get the traffic #/usr/sbin/iftop -Bts ${GSAMPLETIME} -i br0 > "${GFILEOUTPUT}" 2>&1 # for some reason the output goes to stderr cd ~pi /usr/sbin/iftop -Bts ${GSAMPLETIME} > "${GFILEOUTPUT}" 2>&1 # for some reason the output goes to stderr cat "${GFILEOUTPUT}" chown pi:pi "${GFILEOUTPUT}" echo "iftop done. `date`" # just for profiling # Make a copy of the text for later use (maybe) cp "${GFILEOUTPUT}" "${GFOLDERWWWTXT}/${GDATE}_${GTIMEM}.txt" chown pi:pi "${GFOLDERWWWTXT}/${GDATE}_${GTIMEM}.txt" echo "Text-Arch done. `date`" # just for profiling # Create individual logs per host cat "${GFILEOUTPUT}" | grep "=" | grep -v "==" | while read LINE1; do read LINE2 # echo "L1: $LINE1" # echo "L2: $LINE2" LSENDER="`echo ${LINE1} | awk '{printf $2}'`" LSENT="`echo ${LINE1} | awk '{printf $6}'`" # 5=last 10s, 6=last 40s LSENTKB="`makeKBytes ${LSENT}`" LRECEIVER="`echo ${LINE2} | awk '{printf $1}'`" LRECEIVED="`echo ${LINE2} | awk '{printf $5}'`" # 4=last 10s, 5=last 40s LRECEIVEDKB="`makeKBytes ${LRECEIVED}`" # If necessary swap directions to make reading and stacking n plot easier (always internal hosts left, so "up" is really "up") # Wenn SENDER kein "fritz" enthält, aber RECEIVER -> swap! ---> DAS GINGE VIELLEICHT AUCH MIT iftop OPTIONEN! if [ "`echo ${LSENDER} | grep '.fritz.box' > /dev/null ; echo $?`" != 0 -a "`echo ${LSENDER} | grep '192.168' > /dev/null; echo $?`" != 0 ]; then # Didn't find "fritz" on the left #if [ `echo ${LRECEIVER} | grep '.fritz.box' > /dev/null` -o `echo ${LRECEIVER} | grep '192.168' > /dev/null` ]; then # Aber rechts steht ein "fritz" # -> tauschen! TMP="${LSENDER}" LSENDER="${LRECEIVER}" LRECEIVER="${TMP}" TMP="${LSENT}" LSENT="${LRECEIVED}" TMP="${LSENTKB}" LSENTKB="${LRECEIVEDKB}" LRECEIVEDKB="${TMP}" #fi fi # echo "$LSENDER -- $LSENTKB --> <-- $LRECEIVEDKB -- $LRECEIVER" LFILELOG="${GFOLDERLOGS}/${GDATE}_${LSENDER}_${LRECEIVER}.txt" #HIER UNSINN test \! -f "${LFILELOG}" && echo -e "TIME\tSENTKB\tRECEIVEDKB" > "${LFILELOG}" # gnuplot seeams to skip the first line, so make it at least useful echo -e "$GTIME\t$LSENTKB\t$LRECEIVEDKB" >> "${LFILELOG}" chown pi:pi "${LFILELOG}" done echo "Split per host done. `date`" # just for profiling # Log #logger -p daemon.info "${GBASENAME}: Got the stats, begin plotting." # Generate Plot, this can be done by user pi #sudo -u pi /bin/bash /home/pi/bin/trafficPlot.sh > /dev/null 2>&1 sudo -u pi /bin/bash /home/pi/bin/trafficPlot.sh "${GDATE}" "${GTIME}" "${GTIMEM}" echo "Plot script done. `date`" # just for profiling # Log #logger -p daemon.info "${GBASENAME}: Plotting done, removing lock." # Remove Lock test -f "${GFILEOUTPUT}" && rm "${GFILEOUTPUT}" # EOF
trafficPlot.sh
#!/bin/bash #set -x # Purpose: Generate plots for the last 10 minutes # Origin Pi3 # Check if we should suspend operations, then skip the hard part, the plot. Keep logging at all times. /home/pi/bin/confMinLoad.sh >> /dev/null 2>&1 if [ "$?" -eq 0 ]; then logger "`basename $0`: Suspending operations." rm "${GFILELOCK}" exit 0 fi # Globals GPLOTTIME=10 # in minutes GFOLDERTMP="/home/pi/var/trafficCheck/tmp" GFOLDERLOGS="/home/pi/var/trafficCheck/logs" GFOLDERWWW="/var/www/html/traffic/archive" GDATE="${1}" # "`date +%Y-%m-%d`" GTIME="${2}" # "`date +%H:%M`" GTIMEM="${3}" # "`date +%H-%M`" GFILELISTLOGS="${GFOLDERTMP}/loglist.tmp" GFILELISTLOGSTRUNCATED="${GFOLDERTMP}/loglist_truncated.tmp" GFILEPLOTCOMMANDS="${GFOLDERTMP}/plotcommands.txt" #GFILEOUTFORMAT="svg" # 1024x768 is around 15-30K, depending on content GFILEOUTFORMAT="gif" # Around 27K at 1024x768, but merging the two plots takes around <1s instead of 8s as with .svg! #GFILEOUTFORMAT="jpeg" # Around 125K at 1024x768, but merging the two plots takes around <1s instead of 8s as with .svg! GFILEOUT="${GFOLDERWWW}/${GDATE}_${GTIMEM}.${GFILEOUTFORMAT}" #GFILEOUT="${GFOLDERWWW}/${GDATE}_${GTIMEM}.${GFILEOUTFORMAT" GFILEFILELISTTMP="${GFOLDERTMP}/filelist.tmp" GHOURNOW="`date +%H`" GMINOFDAY="`echo ${GTIME} | awk -F: '{ print ($1 * 60) + ($2) }'`" # Calculate the minute of the day (0-1439) #GMINUTESTOKEEP="60" # Minuten des Logs, die dann wirklich an GNUplot verfuettert werden GMINUTESTOKEEP="${GPLOTTIME}" # Minuten des Logs, die dann wirklich an GNUplot verfuettert werden GTIMELOWERWATERMARKMIN="`echo ${GMINOFDAY} - ${GMINUTESTOKEEP} | bc`" if [ "${GTIMELOWERWATERMARKMIN}" -lt 0 ]; then GTIMELOWERWATERMARKMIN=0 fi GTIMELOWERWATERMARK="`echo \"${GTIMELOWERWATERMARKMIN}\" | awk '{ printf \"%02d:%02d\", $1/60, ($1 % 60) }'`" if [ "${GHOURNOW}" -eq 00 ]; then #GHOURLAST="23" ## Stupid" The logs contain only whole days.. GHOURLAST="" else GHOURLAST="`echo ${GHOURNOW} - 1 | bc`" fi function keepLastMinutes() { LFILEPATH="${1}" LMINUTESTOKEEP="`echo ${GMINUTESTOKEEP} + 5 | bc` " # just to be save (and the off-by-one-errors I make...) #LTIMELOWERWATERMARK="`echo ${GMINOFDAY} - ${LMINUTESTOKEEP} | bc`" test -f "${LFILEPATH}.tmp" && rm "${LFILEPATH}.tmp" echo -e "TIME\tSENTKB\tRECEIVEDKB" > "${LFILEPATH}.tmp" # gnuplot seeams to skip the first line, so make it at least useful # Lasse nur die letzten n Minuten stehen #cat "${LFILEPATH}" | while read LLINE; do tail -${LMINUTESTOKEEP} "${LFILEPATH}" | while read LLINE; do LTIME="`echo ${LLINE} | awk '{printf $1}'`" LTIMEMIN="`echo ${LTIME} | awk -F: '{ print ($1 * 60) + ($2) }'`" # minute of day #LUP ="`echo ${LLINE} | awk '{printf $2}'`" #LDOWN="`echo ${LLINE} | awk '{printf $3}'`" if [ "${LTIMEMIN}" -ge "${GTIMELOWERWATERMARKMIN}" ]; then echo "${LLINE}" >> "${LFILEPATH}.tmp" fi done mv "${LFILEPATH}.tmp" "${LFILEPATH}" } function keepLastTwoHours() # THIS IS VERY UGLY AND FLAWED { #test -f "${1}.tmp" && rm "${1}.tmp" echo -e "TIME\tSENTKB\tRECEIVEDKB" > "${1}.tmp" # gnuplot seeams to skip the first line, so make it at least useful if [ -n "${GHOURLAST}" ]; then grep "^${GHOURLAST}" "${1}" >> "${1}.tmp" fi grep "^${GHOURNOW}" "${1}" >> "${1}.tmp" mv "${1}.tmp" "${1}" } # Warte, bis das letzte Log geschrieben ist echo "Init done. `date`" # profiling #sleep 30 # Create upper watermarks # 6Mbit/s = 6000KBit/s = 750KB/s # 576KBit/s = 72KB/s, seltsamerweise messe ich auch mal > 100KByte..., lt. Fritz-Box habe ich 2.2MBit/s Upstream -> 275KB/s #if [ \! -f "${GFOLDERLOGS}/${GDATE}_${GTIMEM}_WATERMARKS.txt" ]; then GMAXWATERMARKFILEPATH="${GFOLDERLOGS}/${GDATE}_MAX_MAX.txt" # echo -e "TIME\tSENTKB\tRECEIVEDKB" > "${GMAXWATERMARKFILEPATH}" # gnuplot seeams to skip the first line, so make it at least useful echo -e "${GTIMELOWERWATERMARK}\t275\t750" > "${GMAXWATERMARKFILEPATH}" echo -e "${GTIME}\t275\t750" >> "${GMAXWATERMARKFILEPATH}" #else # touch "${GFOLDERLOGS}/${GDATE}_${GTIMEM}_WATERMARKS.txt" #fi echo "Upper watermarks done. `date`" # profiling # Get the logs that have changed in the last 10 minutes (of this day) find "${GFOLDERLOGS}" -name "${GDATE}*" -type f -mmin -${GPLOTTIME} -print > "${GFILELISTLOGS}" #cat "${GFILELISTLOGS}" echo "Filelist done. `date`" # profiling # Create truncated versions because we want to plot only the last n minutes test -f "${GFILELISTLOGSTRUNCATED}" && rm "${GFILELISTLOGSTRUNCATED}" if [ 1 -eq 0 ]; then cat "${GFILELISTLOGS}" | while read LINE; do LTRUNCATEDFILEPATH="${GFOLDERTMP}/`echo \"${LINE}\" | rev | cut -d/ -f1 | rev`" if [ 0 -eq 1 ]; then tail -${GPLOTTIME} "${LINE}" > "${LTRUNCATEDFILEPATH}" keepLastTwoHours "${LTRUNCATEDFILEPATH}" else cp "${LINE}" "${LTRUNCATEDFILEPATH}" keepLastMinutes "${LTRUNCATEDFILEPATH}" fi # Check, if it only contains zeros, if yes -> skip it! grep -v SENT "${LTRUNCATEDFILEPATH}" | cut -f2 | grep -v 0 > /dev/null 2>&1 # Check erste spalte (sent) LCOLUMN1ONLYZEROS="$?" grep -v SENT "${LTRUNCATEDFILEPATH}" | cut -f3 | grep -v 0 > /dev/null 2>&1 # Check zweite spalte (rcvd) LCOLUMN2ONLYZEROS="$?" if [ "${LCOLUMN1ONLYZEROS}" -eq 1 -a "${LCOLUMN2ONLYZEROS}" -eq 1 ]; then # Both columns contain only zeros, killing it! #echo "--" #echo "Deleting this log (${LTRUNCATEDFILEPATH}) because it contains only zeros:" #cat "${LTRUNCATEDFILEPATH}" #echo "--" rm "${LTRUNCATEDFILEPATH}" # elif [ "`wc -l \"${LTRUNCATEDFILEPATH}\" | awk '{ printf $1 }'`" -le 1 ]; then # one-line-data files wont work: line 0: warning: Skipping data file with no valid points # echo "--" # echo "Deleting this log (${LTRUNCATEDFILEPATH}) because it contains not enough data:" # cat "${LTRUNCATEDFILEPATH}" # echo "--" else echo "${LTRUNCATEDFILEPATH}" >> "${GFILELISTLOGSTRUNCATED}" fi done else # Do this with PHP - this will save nearly 9 seconds of executiontime (in the evening) /usr/bin/php /home/pi/bin/trafficTruncateLogs.php "${GFILELISTLOGS}" "${GTIMELOWERWATERMARKMIN}" "${GMINUTESTOKEEP}" "${GFILELISTLOGSTRUNCATED}" cat "${GFILELISTLOGSTRUNCATED}" fi #exit 0 echo "Cleaning up done. `date`" # profiling for COLUMN in 3 2; do LFILEOUT="`echo \"${GFILEOUT}\" | rev | cut -d. -f2- | rev`_${COLUMN}.`echo \"${GFILEOUT}\" | rev | cut -d. -f1 | rev`" # Outputfilename per stream direction # Now setup the plot commands # # Init test -f "${GFILEPLOTCOMMANDS}" && rm "${GFILEPLOTCOMMANDS}" echo "reset" >> "${GFILEPLOTCOMMANDS}" echo "set key inside right top vertical Right noreverse enhanced autotitles columnhead nobox" >> "${GFILEPLOTCOMMANDS}" # Set time format for X axis echo "set timefmt \"%H:%M\"" >> "${GFILEPLOTCOMMANDS}" echo "set xdata time" >> "${GFILEPLOTCOMMANDS}" echo "set format x \"%H:%M\"" >> "${GFILEPLOTCOMMANDS}" # Setup line style (#1) for the line #echo "set style line 1 lc rgb \"${LINE1COLOUR}\" pt 1 ps 1 lt 1 lw 2" >> "${GFILEPLOTCOMMANDS}" # http://www.gnuplotting.org/tag/grid/ #echo "set style data lines" >> "${GFILEPLOTCOMMANDS}" # Set X tics (one tic per hour, rotate that tick-labels by 90 deg and move em a bit) #echo "set xtics \"01:00\" rotate by 90 offset 0,-2 out nomirror" >> "${GFILEPLOTCOMMANDS}" # Set Y tics #echo "set ytics (0,1)" >> "${GFILEPLOTCOMMANDS}" # Setup Grid (with line style #12) # echo "set style line 12 lc rgb '#E0E0E0' lt 0 lw 1" >> "${GFILEPLOTCOMMANDS}" # http://www.gnuplotting.org/tag/grid/ #echo "set grid back ls 12" >> "${GFILEPLOTCOMMANDS}" # http://www.gnuplotting.org/tag/grid/ echo "set grid back" >> "${GFILEPLOTCOMMANDS}" # http://www.gnuplotting.org/tag/grid/ # Setup Title #echo "set title \"Traffic Analysis ${GDATE} ${GTIME}\"" >> "${GFILEPLOTCOMMANDS}" if [ "${COLUMN}" -eq 3 ]; then echo "set title \"Downstream Analysis ${GDATE} ${GTIME}\"" >> "${GFILEPLOTCOMMANDS}" else echo "set title \"Upstream Analysis ${GDATE} ${GTIME}\"" >> "${GFILEPLOTCOMMANDS}" fi # Label X and Y Axis echo "set ylabel \"KB/s\"" >> "${GFILEPLOTCOMMANDS}" #echo "set xlabel \"Time\" offset 0,-0.5" >> "${GFILEPLOTCOMMANDS}" echo "set xlabel \"Time\"" >> "${GFILEPLOTCOMMANDS}" echo "set y2tics" >> "${GFILEPLOTCOMMANDS}" # Setup Y range #echo "set yrange [ 0 : 1.1 ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # if [ "${COLUMN}" -eq 3 ]; then # echo "set yrange [ 0 : 750 ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # 6Mbit/s = 6000KBit/s = 750KB/s # echo "set y2range [ 0 : 750 ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # else # echo "set yrange [ 0 : 200 ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # 576KBit/s = 72KB/s, seltsamerweise messe ich auch mal > 100KByte... # echo "set y2range [ 0 : 200 ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # fi # Setup X Range #echo "set xrange [ \"0:00\" : \"24:00\" ] " >> "${GFILEPLOTCOMMANDS}" #echo "set xrange [ \"0:00\" : \"24:00\" ] noreverse nowriteback" >> "${GFILEPLOTCOMMANDS}" # Setup box Style # set boxwidth 0.9 relative # set style fill solid 1.0 # line 0: warning: Skipping data file with no valid points -> set separator, as https://stackoverflow.com/questions/26767359/gnuplot-giving-warning-skipping-data-file-with-no-valid-points suggests set datafile separator "\t" # Sicherheitshalber, eigentlich unnötig # Set output file type to svg and plot it into file echo "set term ${GFILEOUTFORMAT} size 1024,768" >> "${GFILEPLOTCOMMANDS}" echo "set output \"${LFILEOUT}\" " >> "${GFILEPLOTCOMMANDS}" #echo "plot \"${DATAFILE}\" using 1:2 title '${LINE1LABEL}' with l ls 1" >> "${GFILEPLOTCOMMANDS}" #echo "plot \"${DATAFILE}\" using 1:2 title '${LINE1LABEL}' with filledcurves" >> "${GFILEPLOTCOMMANDS}" #echo "plot \"${DATAFILE}\" using 1:2 title '${LINE1LABEL}' with boxes" >> "${GFILEPLOTCOMMANDS}" #echo "plot \"${DATAFILE}\" using 1:2 title '${LINE1LABEL}' with filledcurve x1" >> "${GFILEPLOTCOMMANDS}" echo -n "plot " >> "${GFILEPLOTCOMMANDS}" # Add each line/log cat "${GFILELISTLOGSTRUNCATED}" | while read INFILE; do # Debugging output echo "-" echo "Plotting ${INFILE}: " cat "${INFILE}" # echo "-" LHOST1="`echo \"${INFILE}\" | rev | cut -d/ -f1 | rev | cut -d_ -f2`" LHOST2="`echo \"${INFILE}\" | rev | cut -d/ -f1 | rev | cut -d_ -f3 | sed s/\.tmp// | sed s/\.txt//`" if [ "${COLUMN}" -eq 2 ]; then # Upstream #LLINENAME1="`echo \"${INFILE}\" | rev | cut -d/ -f1 | cut -d_ -f1-2 | rev` sent" LLINENAME1="${LHOST1} --> ${LHOST2}" echo -n "\"${INFILE}\" using 1:2 title '${LLINENAME1}' with linespoint, " >> "${GFILEPLOTCOMMANDS}" #echo -n "\"${INFILE}\" using 1:2 title '${LLINENAME1}' with boxes, " >> "${GFILEPLOTCOMMANDS}" else # Downstream #LLINENAME2="`echo \"${INFILE}\" | rev | cut -d/ -f1 | cut -d_ -f1-2 | rev` received" LLINENAME2="${LHOST1} <-- ${LHOST2}" echo -n "\"${INFILE}\" using 1:3 title '${LLINENAME2}' with linespoint, " >> "${GFILEPLOTCOMMANDS}" fi done # Plot! # cat "${GFILEPLOTCOMMANDS}" | rev | cut -c3- | rev truncate -s-2 "${GFILEPLOTCOMMANDS}" # das ", " abschneiden, das von der vorherigen Schleife übrig blieb # cat "${GFILEPLOTCOMMANDS}" cat "${GFILEPLOTCOMMANDS}" | gnuplot # Since we added one "," too much, we must leave it out done # end loop for columns echo "Plotting done. `date`" # profiling # Merge the Images #find "${GFOLDERWWW}" -maxdepth 1 -type f | grep -v last | sort | tail -2 montage -verbose -mode concatenate -tile 1x "${GFOLDERWWW}/${GDATE}_${GTIMEM}_3.${GFILEOUTFORMAT}" "${GFOLDERWWW}/${GDATE}_${GTIMEM}_2.${GFILEOUTFORMAT}" "${GFILEOUT}" rm "${GFOLDERWWW}/${GDATE}_${GTIMEM}_3.${GFILEOUTFORMAT}" "${GFOLDERWWW}/${GDATE}_${GTIMEM}_2.${GFILEOUTFORMAT}" echo "Merging plots done. `date`" # profiling # Move into place cp "${GFILEOUT}" "${GFOLDERWWW}/lastsnap.${GFILEOUTFORMAT}" echo "Copy done. `date`" # profiling #exit 0 # Cleanup tmp cat "${GFILELISTLOGSTRUNCATED}" | while read INFILE; do rm "${INFILE}" done echo "Cleanup truncated files done. `date`" # profiling # Keep the last 10 Minutes (svgs), move the rest into archive-Folder # ls -1 "${GFOLDERWWW}" | sort > "${GFILEFILELISTTMP}" find "${GFOLDERWWW}" -maxdepth 1 -type f | sort > "${GFILEFILELISTTMP}" C="`wc -l \"${GFILEFILELISTTMP}\" | awk '{printf $1}'`" D="`echo $C - ${GPLOTTIME} | bc`" if [ "${C}" -gt "${GPLOTTIME}" ]; then mkdir -p "${GFOLDERWWW}/${GDATE}" head -$D "${GFILEFILELISTTMP}" | while read LINE; do echo "Archiving $LINE..." mv "$LINE" "${GFOLDERWWW}/${GDATE}/" #mv "${GFOLDERWWW}/$LINE" "${GFOLDERWWW}/${GDATE}/" done fi echo "Archiving old plots done. `date`" # profiling # EOF
trafficTruncateLogs.php
<?php /* Script: trafficTruncateLogs.php Purpose: create truncated logs for trafficPlot.sh, replacing that code in trafficPlot.sh (which is very slow) Origin: Pi3 /* Globals */ $gLogsListPath=$argv[1]; $gGTimeLowerWatermarkMinutes=$argv[2]; $gMinutesToKeep=$argv[3]; $gLogsListPathShortened=$argv[4]; #print_r($argv); /* Main */ /* Read the Original Log-List-File */ $logListString=file_get_contents($gLogsListPath); $logListArray=explode("\n", $logListString); /* Walk thru lines */ foreach ($logListArray as $logListOnePath) { if (strlen($logListOnePath)>5) { echo ("Working on $logListOnePath...\n"); /* Create shortened copy of the logfile */ $logListOnePathShortened=$logListOnePath . ".tmp"; $cmd="tail -" . ($gMinutesToKeep + 5) . " \"$logListOnePath\" > \"$logListOnePathShortened\""; #echo ("cmd: $cmd\n"); system ($cmd); /* LOL: https://stackoverflow.com/questions/15025875/what-is-the-best-way-in-php-to-read-last-lines-from-a-file/15025877 */ /* Now Walk thru the shortened one, shorten it even more, and check whether it contains only 0 or not */ $oneLogString=file_get_contents($logListOnePathShortened); $oneLogArray=explode("\n", $oneLogString); $oneLogArrayNew=array(); $foundNonZero=false; array_unshift($oneLogArray, "TIME\tSENTKB\tRECEIVEDKB"); # gnuplot seeams to skip the first line, so make it at least useful foreach ($oneLogArray as $oneLogLine) { if (strpos($oneLogLine, "SENT") === false && strlen($oneLogLine) > 5 ) // TBD: Match Line /beginning/ with # { $oneLogLineArray=explode("\t", $oneLogLine); $time=$oneLogLineArray[0]; $up=$oneLogLineArray[1]; $down=$oneLogLineArray[2]; /* Skip all lines befor given date */ //awk -F: '{ print ($1 * 60) + ($2) $timeArray=explode(":", $time); #print_r($timeArray); $timeInMinutes=$timeArray[0] * 60 + $timeArray[1]; if ($timeInMinutes >= $gGTimeLowerWatermarkMinutes) { /* Check for all-zero-file */ if ($up != 0 || $down != 0) { echo ("\"$up\" and \"$down\", nonzero\n"); $foundNonZero=true; } /* Add line */ array_push($oneLogArrayNew, $oneLogLine); } } // if comment else { array_push($oneLogArrayNew, $oneLogLine); } } // for each shortened log file line /* Write Array back to file, or even kill the file */ if ($foundNonZero == true) { echo("Overwriting shortened log with:\n"); print_r($oneLogArrayNew); /* Overwrite with (even more) shortened log */ file_put_contents($logListOnePathShortened, implode("\n", $oneLogArrayNew)); /* Remind it for later */ echo("Adding \"$logListOnePathShortened\" to \"$gLogsListPathShortened\"\n"); file_put_contents($gLogsListPathShortened, $logListOnePathShortened . "\n", FILE_APPEND); } else { unlink($logListOnePathShortened); } } // plausi logListOnePath } // foreach original log list ?>
And this is what you get...
hogwatch
Installation
pip install hogwatch --upgrade
Start
sudo hogwatch server
A really nice looking web page. WAF ok.
I abandoned this tool, since it doesn't work for me.
nethogs
TBD