testing: generate HTML version of .log files and ClientTest.cpp

The HTML version of the .log files links to ClientTest.cpp.html, colorizes important parts and links to the sync session directories. Because it is much more useful than the plain text version, the nightly.html now links to these .html files. ClientTest.cpp.html is built with Python pygments if installed, otherwise some builtin fallback code is used.
author: Patrick Ohly <patrick.ohly@intel.com> 2011-11-08 12:57:24 +0100
committer: Patrick Ohly <patrick.ohly@intel.com> 2011-11-09 10:37:05 +0000
commit: 37f360d33da1936d1971d0351b4167a316658591 (patch)
tree: ece43812659113e498f896fb358cb344d402c3cd
parent: 2f39cb1f7a03799d15c6be488dfa6afd47c4e823 (diff)
9 files changed, 183 insertions, 5 deletions
diff --git a/build/build.am b/build/build.am
index 5f0bdce5..09309466 100644
--- a/build/build.am
+++ b/build/build.am
@@ -1 +1,2 @@
-dist_noinst_SCRIPTS += build/gen-git-version.sh
+dist_noinst_SCRIPTS += build/gen-git-version.sh \
+	build/source2html.py
diff --git a/build/source2html.py b/build/source2html.py
new file mode 100644
index 00000000..ea7dca5a
--- /dev/null
+++ b/build/source2html.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+
+"""
+Converts source code (first parameter, can be - for stdin) to HTML
+(stdout), using pygments if installed, otherwise simple text
+manipulation without syntax highlighting. In both cases the output
+will have "True-<number>" as anchors for each line.
+"""
+
+import sys
+
+filename = sys.argv[1]
+if filename == '-':
+    code = sys.stdin.read()
+else:
+    code = open(filename).read()
+
+out = sys.stdout
+
+try:
+    import pygments
+    import pygments.lexers
+    from pygments.formatters import HtmlFormatter
+    if filename == '-':
+        lexer = pygments.lexers.guess_lexer(code)
+    else:
+        lexer = pygments.lexers.guess_lexer_for_filename(filename, code)
+    formatter = HtmlFormatter(full=True, linenos=True, lineanchors=True)
+    pygments.highlight(code, lexer, formatter, out)
+except:
+    import cgi
+    print >>sys.stderr, "source2html.py failed with pygments:", sys.exc_info()
+    print >>sys.stderr, "falling back to internal code"
+
+    out.write('''<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+   <meta http-equiv="content-type" content="text/html; charset=None">
+   <style type="text/css">
+td.linenos { background-color: #f0f0f0; padding-right: 10px; }
+span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
+pre { line-height: 125%; }
+body .hll { background-color: #ffffcc }
+body .hll { background-color: #ffffcc }
+body  { background: #f8f8f8; }
+   </style>
+</head>
+<body>
+<table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre>''')
+    lines = code.split('\n')
+    for line in range(1, len(lines)):
+        out.write('%4d\n' % line)
+    out.write('%4d' % len(lines))
+    out.write('</pre></div></td><td class="code"><div class="highlight"><pre>')
+    for lineno, line in enumerate(lines):
+        out.write('<a name="True-%d"></a>%s\n' % (lineno + 1, cgi.escape(line)))
+    out.write('''</pre></div>
+</td></tr></table></body>
+</html>''')
diff --git a/setup-variables.am b/setup-variables.am
index ffc3df18..1fe4ecae 100644
--- a/setup-variables.am
+++ b/setup-variables.am
@@ -23,6 +23,7 @@ dist_doc_DATA =
 dist_noinst_DATA =
 dist_pkgdata_DATA =
 doc_DATA =
+nodist_noinst_DATA =
 
 lib_LTLIBRARIES =
 noinst_LTLIBRARIES =
diff --git a/src/src.am b/src/src.am
index 516ff2a2..25f7446f 100644
--- a/src/src.am
+++ b/src/src.am
@@ -65,6 +65,12 @@ src/synccompare : $(top_srcdir)/test/Algorithm/Diff.pm $(top_srcdir)/test/syncco
 	$(AM_V_GEN)perl -e '$$diff = shift; open(DIFF, "<$$diff"); ($$_) = split(/__END__/, join("", <DIFF>)); s/\*import.*//m; s/require +Exporter;//; s/^#.*\n//mg; s/ +#.*\n//mg; $$diff = $$_;' -e 'while(<>) {' @MODIFY_SYNCCOMPARE@ -e 's/use +Algorithm::Diff;/"# embedded version of Algorithm::Diff follows, copyright by the original authors\n" . $$diff . "# end of embedded Algorithm::Diff\n"/e; print;}' $+ >$@ \
 	&&chmod u+x $@
 
+# helper script for testing
+bin_SCRIPTS += src/synclog2html
+CLEANFILES += src/synclog2html
+src/synclog2html: $(top_srcdir)/test/log2html.py
+	$(AM_V_GEN)cp $< $@ && chmod u+x $@
+
 CORE_SOURCES =
 
 # The files which register backends have to be compiled into
@@ -153,7 +159,7 @@ clean-local: testclean
 
 # files created during testing
 testclean:
-	rm -rf src/*.test.vcf src/*.log src/*.tests src/*.diff src/*.dat src/*Client_Sync_*client.*
+	rm -rf src/*.test.vcf src/*.log src/*.tests src/*.diff src/*.dat src/*Client_Sync_*client.* src/*Client_Source*
 
 distclean-local:
 	rm -rf $(SYNTHESIS_SUBDIR)
@@ -294,6 +300,13 @@ testcase2patch: $(TEST_FILES_GENERATED)
 	  fi; \
 	done
 
+# generate syntax-highlighted version of ClientTest.cpp for HTML
+# version of .log test results
+nodist_noinst_DATA += src/ClientTest.cpp.html
+CLEANFILES += src/ClientTest.cpp.html
+src/ClientTest.cpp.html: build/source2html.py test/ClientTest.cpp
+	$(AM_V_GEN)python $+ >$@
+
 # copy base test files
 $(filter-out %.tem, $(filter src/testcases/%, $(subst $(top_srcdir)/test/,src/,$(CLIENT_LIB_TEST_FILES)))) : src/% : $(top_srcdir)/test/%
 	$(AM_V_at)mkdir -p '$(dir $@)'; \
@@ -302,7 +315,7 @@ $(filter-out %.tem, $(filter src/testcases/%, $(subst $(top_srcdir)/test/,src/,$
 # The binary does not really depend on the test cases, only running it does.
 # Listing the dependencies here is done to ensure that one doesn't accidentally
 # runs the binary with out-dated auxiliary files.
-src_client_test_DEPENDENCIES = $(EXTRA_LTLIBRARIES) $(CORE_DEP) $(CLIENT_LIB_TEST_FILES) testcase2patch src/synccompare src/templates
+src_client_test_DEPENDENCIES = $(EXTRA_LTLIBRARIES) $(CORE_DEP) $(CLIENT_LIB_TEST_FILES) testcase2patch src/synccompare src/synclog2html src/templates
 
 CLEANFILES += $(src_client_test_buteo_moc_files)
 
diff --git a/test/client-test-main.cpp b/test/client-test-main.cpp
index 31cc47bd..fe777b81 100644
--- a/test/client-test-main.cpp
+++ b/test/client-test-main.cpp
@@ -199,6 +199,11 @@ public:
             }
         }
 
+        std::string htmllog = logfile + ".html";
+        system(StringPrintf("synclog2html %s >%s",
+                            logfile.c_str(),
+                            htmllog.c_str()).c_str());
+
         std::cout << " " << result << "\n";
         if (!failure.empty()) {
             std::cout << failure << "\n";
diff --git a/test/generate-html.xsl b/test/generate-html.xsl
index b6355fe5..0b7354c4 100644
--- a/test/generate-html.xsl
+++ b/test/generate-html.xsl
@@ -34,7 +34,7 @@
     <xsl:variable name="invalid-value" select="'invalid-value'"/>
 
     <!-- log file suffix name -->
-    <xsl:variable name="log-file-suffix" select="'.log'"/>
+    <xsl:variable name="log-file-suffix" select="'.html'"/>
 	
     <xsl:template match="/">
         <xsl:choose>
diff --git a/test/log2html.py b/test/log2html.py
new file mode 100644
index 00000000..7f2a1d08
--- /dev/null
+++ b/test/log2html.py
@@ -0,0 +1,98 @@
+#!/usr/bin/python
+
+"""
+Converts the .log output for a client-test test into HTML, with
+hightlighting and local hrefs to ClientTest.html and test directories.
+"""
+
+import sys
+import re
+import cgi
+
+filename = sys.argv[1]
+if filename == '-':
+    log = sys.stdin
+else:
+    log = open(filename)
+
+out = sys.stdout
+
+# matches [DEBUG/DEVELOPER/...] tags at the start of the line,
+# used to mark text via <span class=mode>
+class ModeSpan:
+    mode=None
+
+    @staticmethod
+    def setMode(newmode):
+        if ModeSpan.mode != newmode:
+            if ModeSpan.mode:
+                out.write('</span>')
+            ModeSpan.mode = newmode
+            if ModeSpan.mode:
+                out.write('<span class="%s">' % ModeSpan.mode)
+
+# detect SyncEvolution line prefix
+tag = re.compile(r'^(\[([A-Z]+) [^\]]*\])( .*)')
+
+# hyperlink to HTML version of source code
+sourcefile = re.compile(r'((\w+\.(?:cpp|c|h)):(\d+))')
+
+# highlight:
+# - any text after ***
+#   *** clean via source A
+#   *** starting Client::Source::egroupware-dav_caldav::testChanges ***
+# - simple prefixes at the start of a line, after the [] tag (removed already)
+#    caldav #A:
+# - HTTP requests
+#   REPORT /egw/groupdav.php/syncevolution/calendar/ HTTP/1.1
+#   PROPFIND /egw/groupdav.php/syncevolution/calendar/1234567890%40dummy.ics HTTP/1.1
+highlight = re.compile(r'(\*\*\* .*|^ [a-zA-Z0-9_\- #]*: |(?:REPORT|PUT|GET|DELETE|PROPFIND) .*HTTP/\d\.\d$)')
+
+# hyperlink to sync session directory
+session = re.compile(r'(Client_(?:Source|Sync)(?:_\w+)+\S+)')
+
+out.write('''<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+   <meta http-equiv="content-type" content="text/html; charset=None">
+   <style type="text/css">
+td.linenos { background-color: #f0f0f0; padding-right: 10px; }
+span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
+pre { line-height: 125%; }
+body .hll { background-color: #ffffcc }
+body  { background: #f8f8f8; }
+span.INFO { background: #c0c0c0 }
+span.ERROR { background: #e0c0c0 }
+span.hl { color: #c02020 }
+   </style>
+</head>
+<body>
+<pre>''')
+
+def simplifyFilename(test):
+    "same as client-test-main.cpp simplifyFilename()"
+    test = test.replace(':', '_')
+    test = test.replace('__', '_')
+    return test
+
+for line in log:
+    line = line.rstrip()
+    m = tag.match(line)
+    if m:
+        ModeSpan.setMode(m.group(2))
+        out.write('<span class="tag">%s</span>' % m.group(1))
+        line = m.group(3)
+    line = cgi.escape(line)
+    line = sourcefile.sub(r'<a href="\2.html#True-\3">\1</a>', line)
+    line = highlight.sub(r'<span class="hl">\1</span>', line)
+    line = session.sub(r'<a href="\1/">\1</a>', line)
+    out.write(line)
+    out.write('\n')
+
+# close any <span class=mode> opened before
+ModeSpan.setMode(None)
+
+out.write('''
+</pre>
+</body>
+</html>''')
diff --git a/test/runtests.py b/test/runtests.py
index 2b034b0c..2b88aa64 100755
--- a/test/runtests.py
+++ b/test/runtests.py
@@ -612,7 +612,7 @@ class SyncEvolutionTest(Action):
             else:
                 context.runCommand(basecmd)
         finally:
-            tocopy = re.compile(r'.*\.log|.*\.client.[AB]')
+            tocopy = re.compile(r'.*\.log|.*\.client.[AB]|.*\.(cpp|h|c)\.html|.*\.log\.html')
             htaccess = file(os.path.join(resdir, ".htaccess"), "a")
             for f in os.listdir(self.srcdir):
                 if tocopy.match(f):
diff --git a/test/test.am b/test/test.am
index 022fc14c..ba6096f8 100644
--- a/test/test.am
+++ b/test/test.am
@@ -12,6 +12,7 @@ dist_noinst_SCRIPTS += \
   test/syncevo-http-server.py \
   test/syncevo-phone-config.py \
   test/synccompare.pl \
+  test/log2html.py \
   test/run_src_client_test.sh
 
 dist_noinst_DATA += \
author	Patrick Ohly <patrick.ohly@intel.com>	2011-11-08 12:57:24 +0100
committer	Patrick Ohly <patrick.ohly@intel.com>	2011-11-09 10:37:05 +0000
commit	37f360d33da1936d1971d0351b4167a316658591 (patch)
tree	ece43812659113e498f896fb358cb344d402c3cd
parent	2f39cb1f7a03799d15c6be488dfa6afd47c4e823 (diff)