From 4b88f15929d3d4c92449b32c9426f68e4e8b4ef6 Mon Sep 17 00:00:00 2001 From: Mark Linimon Date: Thu, 26 Mar 2009 10:33:51 +0000 Subject: Rewrite of processlog by kris. --- Tools/portbuild/scripts/processlogs | 231 +++++++++++++++++++++++------------- 1 file changed, 147 insertions(+), 84 deletions(-) (limited to 'Tools') diff --git a/Tools/portbuild/scripts/processlogs b/Tools/portbuild/scripts/processlogs index e0b86b4b7ee9..53dbdcefe712 100755 --- a/Tools/portbuild/scripts/processlogs +++ b/Tools/portbuild/scripts/processlogs @@ -1,4 +1,19 @@ #!/bin/sh + +# Process the logs in a certain directory and construct the HTML pages +# summarizing them +# +# We only construct the new html pages if either +# * No HTML files exist yet +# * .force exists in this directory (set by pdispatch when a log is +# removed by package build success) +# * A new log file has appeared since the last run. +# +# Individual summary data for the logs is saved between runs instead of +# being recomputed each time. Removing the old data for logs that +# no longer exist is the responsibility of the dopackages script when +# starting a new build. + # echo 'processlogs: at '`date`', begin' arch=$1 @@ -6,14 +21,18 @@ arch=$1 # establish which directory the logfiles live in (leave out for # backwards compatibility) buildlogdir=$(realpath .) -if [ "$2" != "" ]; then buildlogdir="$2"; fi +if [ "$2" != "" ]; then + buildlogdir="$2" +fi baselogdir=$(basename ${buildlogdir}) # establish which directory INDEX lives in (leave out for # backwards compatibility) indexlogdir=$(realpath .) -if [ "$3" != "" ]; then indexlogdir="$3"; fi +if [ "$3" != "" ]; then + indexlogdir="$3"; +fi # allow this script to be run from anywhere in the tree scriptdir=$(dirname $0) @@ -24,18 +43,29 @@ pb=/var/portbuild . ${pb}/${arch}/portbuild.conf -# create a name for the tempfile -of=.index.html +# get the list of buildlogs. -# if there are no new logfiles, there is nothing to do here. -if [ \! -e .force -a -e .stamp -a $(echo $(find -f $buildlogdir -maxdepth 1 -newer .stamp -type f \( -name '*.log' -o -name '*.log.bz2' \) 2>/dev/null | wc -l)) = "0" ]; then exit; fi +cd ${buildlogdir} -rm -f .force -touch .stamp +if [ -e .force -o \! -e index.html ]; then + rm -f .force + force=1 +else + force=0 +fi -# get the list of buildlogs. -set $buildlogdir/*.log $buildlogdir/*.log.bz2 -> .logs +alllogs=$(find . -maxdepth 1 -type f \( -name '*.log' -o -name '*.log.bz2' \) | sed -e 's,^./,,' 2>/dev/null) + +if [ -e .stamp -a ${#alllogs} -eq 0 ]; then + exit +fi + +# create a name for the tempfile +of=.index.html + +# XXX temporary +test -f .logs && rm -f .logs +mkdir -p .logs # # Read the log-files and write summaries to .logs in the format @@ -43,24 +73,45 @@ set $buildlogdir/*.log $buildlogdir/*.log.bz2 # $reason|$tag|$broken|$datetime # -# echo 'processlogs: at '`date`', begin processing log files' +echo "processlogs: at $(date), begin processing log files for ${arch} in ${baselogdir}" + +rm -f .logs/.all +set -- ${alllogs} -if [ $# != 1 -o "x$1" != "x`dirname $1`/*.log" -o "x$1" != "x`dirname $1`/*.log.bz2" ]; then - while [ ! -z "$1" ]; do - filename=$(basename $1) - #echo 'processlogs: at '`date`', examining '$filename' +gotnew=0 - $errorscript $filename $indexlogdir >> .logs +while [ $# -ge 1 ]; do + log=$1 shift - done -fi -# echo 'processlogs: at '`date`', end processing log files' -# XXX Sometimes log entries get doubled up for some reason -uniq .logs > .logs2 -mv .logs2 .logs + if [ ${log%.log.bz2} != ${log} -a -e ${log%.bz2} ]; then + # We have both a .log.bz2 and a .log, assume the former is stale + rm -f ${log} + continue + fi + + # basename with .log and .log.bz2 stripped + base=${log%.bz2} + base=${log%.log} -num=$(wc -l < .logs) + if [ ${log} -nt .stamp -o ! -e .logs/${base} ]; then + # Save to the base name so the data persists after the log is + # compressed + ${errorscript} ${log} ${indexlogdir} > .logs/${base} + gotnew=1 + fi + + cat .logs/${base} >> .logs/.all +done + +touch .stamp + +echo "processlogs: at "`date`", end processing log files for ${arch} in ${baselogdir}" + +if [ ${force} -eq 0 -a ${gotnew} -eq 0 ]; then + # Nothing new to do + exit +fi header() { echo "Package building errors" >$of @@ -73,17 +124,17 @@ header() { echo "| builddate " >>$of echo "]

" >>$of - if [ $num = "0" ]; then + if [ ${#alllogs} = "0" ]; then echo "No errors (yet)" >>$of else if [ -s cvsdone ]; then echo "CVS update finished at: $(cat cvsdone)
" >> $of fi - latest=$(cd $buildlogdir; ls -rtTl *.log *.log.bz2 | tail -1 | awk '{printf("%s %s %s %s\n",$6,$7,$8,$9)}';) + latest=$(ls -rtTl *.log *.log.bz2 2> /dev/null | tail -1 | awk '{printf("%s %s %s %s\n",$6,$7,$8,$9)}') echo "Timestamp of newest log: $latest

" >> $of echo "\"Aff.\" is number of ports that depend on this one
" >> $of echo "\"[B]\" indicates port is marked BROKEN (Note: BROKEN ports are not frequently rebuilt so they may not be listed here)

" >> $of - echo "

$num errors

" >> $of + echo "

${#alllogs} errors

" >> $of echo "" >>$of echo "$1" >>$of fi @@ -95,10 +146,7 @@ footer() { echo "" >>$of } -# Now reread the .logs file and create the reports. If the .logs file -# has blanks in any field, this code won't work. Therefore, the -# code above has to guarantee that that won't happen, and the code -# below has to catch the metatoken. +# Now reread the .logs/.all file and create the reports. # echo 'processlogs: at '`date`', create default output' # @@ -106,10 +154,13 @@ footer() { # header "" -for i in `cat .logs | sort`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") +if [ ${#alllogs} -gt 0 ]; then +sort .logs/.all | while read line; do + IFS="|" + set -- ${line} + unset IFS mailto="$6" - if [ "$6" != " " ] ; then + if [ "$6" != "" ] ; then mailto="$6" fi @@ -117,23 +168,24 @@ for i in `cat .logs | sort`; do echo "" >> $of - affby=$3 - test $affby = "0" -o $affby = "-1" && affby=" " - echo "" >> $of + affby="$3" + test "${affby}" = "0" -o "${affby}" = "-1" && affby=" " + echo "" >> $of echo "" >> $of echo "" >> $of echo "" >> $of - date=$(echo ${10} | tr '_' ' ') + date="${10}" echo "" >> $of echo "" >> $of done +fi footer "" mv -f $of index.html @@ -143,10 +195,13 @@ mv -f $of index.html # header "" -for i in `cat .logs | sort -t \\| +4`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") +if [ ${#alllogs} -gt 0 ]; then +sort -t \| +4 .logs/.all | while read line; do + IFS="|" + set -- $line + unset IFS mailto="$6" - if [ "$6" != " " ] ; then + if [ "$6" != "" ] ; then mailto="$6" fi @@ -154,23 +209,24 @@ for i in `cat .logs | sort -t \\| +4`; do echo "" >> $of - affby=$3 - test $affby = "0" -o $affby = "-1" && affby=" " - echo "" >> $of + affby="$3" + test "${affby}" = "0" -o "${affby}" = "-1" && affby=" " + echo "" >> $of echo "" >> $of echo "" >> $of echo "" >> $of - date=$(echo ${10} | tr '_' ' ') + date="${10}" echo "" >> $of echo "" >> $of done +fi footer "" mv -f $of index-category.html @@ -180,10 +236,13 @@ mv -f $of index-category.html # header "" -for i in `cat .logs | sort -t \\| +5`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") +if [ ${#alllogs} -gt 0 ]; then +sort -t \| +5 .logs/.all | while read line; do + IFS="|" + set -- $line + unset IFS mailto="$6" - if [ "$6" != " " ] ; then + if [ "$6" != "" ] ; then mailto="$6" fi @@ -192,22 +251,23 @@ for i in `cat .logs | sort -t \\| +5`; do echo "" >> $of echo "" >> $of - affby=$3 - test $affby = "0" -o $affby = "-1" && affby=" " - echo "" >> $of + affby="$3" + test "${affby}" = "0" -o "${affby}" = "-1" && affby=" " + echo "" >> $of echo "" >> $of echo "" >> $of - date=$(echo ${10} | tr '_' ' ') + date="${10}" echo "" >> $of echo "" >> $of done +fi footer "" mv -f $of index-maintainer.html @@ -217,10 +277,13 @@ mv -f $of index-maintainer.html # header "" -for i in `cat .logs | sort -t \\| +7`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") +if [ ${#alllogs} -gt 0 ]; then +sort -t \| +7 .logs/.all | while read line; do + IFS="|" + set -- ${line} + unset IFS mailto="$6" - if [ "$6" != " " ] ; then + if [ "$6" != "" ] ; then mailto="$6" fi @@ -228,23 +291,24 @@ for i in `cat .logs | sort -t \\| +7`; do echo "" >> $of echo "" >> $of - affby=$3 - test $affby = "0" -o $affby = "-1" && affby=" " - echo "" >> $of + affby="$3" + test "${affby}" = "0" -o "${affby}" = "-1" && affby=" " + echo "" >> $of echo "" >> $of echo "" >> $of - date=$(echo ${10} | tr '_' ' ') + date="${10}" echo "" >> $of echo "" >> $of done +fi footer "" mv -f $of index-reason.html @@ -254,50 +318,49 @@ mv -f $of index-reason.html # header "" -for i in `cat .logs | sort -t \\| +9`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") +if [ ${#alllogs} -gt 0 ]; then +sort -t \| +9 .logs/.all | while read line; do + IFS="|" + set -- ${line} + unset IFS + mailto="$6" - if [ "$6" != " " ] ; then + if [ "$6" != "" ] ; then mailto="$6" fi echo "" >> $of - date=$(echo ${10} | tr '_' ' ') + date="${10}" echo "" >> $of echo "" >> $of - affby=$3 - test $affby = "0" -o $affby = "-1" && affby=" " - echo "" >> $of + affby="$3" + test "${affby}" = "0" -o "${affby}" = "-1" && affby=" " + echo "" >> $of echo "" >> $of echo "" >> $of echo "" >> $of echo "" >> $of done +fi footer "" mv -f $of index-builddate.html # echo 'processlogs: at '`date`', create maintainer list' # # Get list of maintainers. -# -for i in `cat .logs | sort -t \\| +9`; do - set $(echo "$i" | tr \| " " | sed -e "s@NONE@\\ @g") - mailto="$6" - if [ "$6" != " " ] ; then - mailto="$6" - fi - - maints="$maints $6" -done +if [ ${#alllogs} -gt 0 ]; then + cut -f 6 -d \| .logs/.all | sort -fu > maintainers +else + cat /dev/null > maintainers +fi -echo $maints | sed -e 's/ /\ -/g' | sort -fu > maintainers +# echo 'processlogs: at '`date`', done' -- cgit v1.2.3
PortAff.SizeCVSMaintainerReasonBuild date$2$affby$4 Kb${affby}$4 Kb$5$mailto" >> $of - test "$9" = "broken" && echo "[B]" >> $of - reason=$(echo $7 | tr '_' ' ') + test "$9" = "yes" && echo "[B]" >> $of + reason="$7" echo "$reason" >> $of echo "$date
CVSAff.SizePortMaintainerReasonBuild date$5$affby$4 Kb${affby}$4 Kb$2$mailto" >> $of test "$9" = "broken" && echo "[B]" >> $of - reason=$(echo $7 | tr '_' ' ') + reason="$7" echo "$reason" >> $of echo "$date
MaintainerPortAff.SizeCVSReasonBuild date$mailto$2$affby$4 Kb${affby}$4 Kb$5" >> $of test "$9" = "broken" && echo "[B]" >> $of - reason=$(echo $7 | tr '_' ' ') + reason="$7" echo "$reason" >> $of echo "$date
ReasonPortAff.SizeCVSMaintainerBuild date" >> $of test "$9" = "broken" && echo "[B]" >> $of - reason=$(echo $7 | tr '_' ' ') + reason="$7" echo "$reason" >> $of echo "$2$affby$4 Kb${affby}$4 Kb$5$mailto$date
Build datePortAff.SizeCVSMaintainerReason
$date$2$affby$4 Kb${affby}$4 Kb$5$mailto" >> $of test "$9" = "broken" && echo "[B]" >> $of - reason=$(echo $7 | tr '_' ' ') + reason="$7" echo "$reason" >> $of echo "