Enables output in the JUnit XML format.

Adds basic support for optionally outputting in the XML format
commonly used by JUnit compatible tools.

This format is supported by default by many tools, including
the Jenkins build system. It also is more detailed and
captures more information than the more simplistic TAP
format.

Signed-off-by: Jon A. Cruz <jonc@osg.samsung.com>
Reviewed-by: Pekka Paalanen <pekka.paalanen@collabora.co.uk>
dev
Jon A. Cruz 9 years ago committed by Pekka Paalanen
parent 315476fa7b
commit 646aef543e
  1. 9
      Makefile.am
  2. 25
      configure.ac
  3. 470
      tools/zunitc/src/zuc_junit_reporter.c
  4. 38
      tools/zunitc/src/zuc_junit_reporter.h
  5. 17
      tools/zunitc/src/zunitc_impl.c

@ -984,6 +984,8 @@ libzunitc_la_SOURCES = \
tools/zunitc/src/zuc_context.h \
tools/zunitc/src/zuc_event.h \
tools/zunitc/src/zuc_event_listener.h \
tools/zunitc/src/zuc_junit_reporter.c \
tools/zunitc/src/zuc_junit_reporter.h \
tools/zunitc/src/zuc_types.h \
tools/zunitc/src/zunitc_impl.c \
shared/helpers.h
@ -995,6 +997,13 @@ libzunitc_la_CFLAGS = \
libzunitc_la_LIBADD = \
libshared.la
if ENABLE_JUNIT_XML
libzunitc_la_CFLAGS += \
$(LIBXML2_CFLAGS)
libzunitc_la_LIBADD += \
$(LIBXML2_LIBS)
endif
libzunitcmain_la_SOURCES = \
tools/zunitc/src/main.c

@ -426,6 +426,30 @@ if test "x$enable_dbus" != "xno"; then
fi
AM_CONDITIONAL(ENABLE_DBUS, test "x$enable_dbus" = "xyes")
# Note that other features might want libxml2, or this feature might use
# alternative xml libraries at some point. Therefore the feature and
# pre-requisite concepts are split.
AC_ARG_ENABLE(junit_xml,
AS_HELP_STRING([--disable-junit-xml],
[do not build with JUnit XML output]),,
enable_junit_xml=auto)
if test "x$enable_junit_xml" != "xno"; then
PKG_CHECK_MODULES(LIBXML2,
[libxml-2.0 >= 2.6],
have_libxml2=yes,
have_libxml2=no)
if test "x$have_libxml2" = "xno" -a "x$enable_junit_xml" = "xyes"; then
AC_MSG_ERROR([JUnit XML support explicitly requested, but libxml2 couldn't be found])
fi
if test "x$have_libxml2" = "xyes"; then
enable_junit_xml=yes
AC_DEFINE(ENABLE_JUNIT_XML, [1], [Build Weston with JUnit output support])
else
enable_junit_xml=no
fi
fi
AM_CONDITIONAL(ENABLE_JUNIT_XML, test "x$enable_junit_xml" = "xyes")
# ivi-shell support
AC_ARG_ENABLE(ivi-shell,
AS_HELP_STRING([--disable-ivi-shell],
@ -537,6 +561,7 @@ AC_MSG_RESULT([
FBDEV Compositor ${enable_fbdev_compositor}
RDP Compositor ${enable_rdp_compositor}
Screen Sharing ${enable_screen_sharing}
JUnit XML output ${enable_junit_xml}
Raspberry Pi BCM headers ${have_bcm_host}

@ -0,0 +1,470 @@
/*
* Copyright © 2015 Samsung Electronics Co., Ltd
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "config.h"
#include "zuc_junit_reporter.h"
#if ENABLE_JUNIT_XML
#include <fcntl.h>
#include <libxml/parser.h>
#include <memory.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include "zuc_event_listener.h"
#include "zuc_types.h"
#include "shared/zalloc.h"
/**
* Hardcoded output name.
* @todo follow-up with refactoring to avoid filename hardcoding.
* Will allow for better testing in parallel etc. in general.
*/
#define XML_FNAME "test_detail.xml"
#define ISO_8601_FORMAT "%Y-%m-%dT%H:%M:%SZ"
/**
* Internal data.
*/
struct junit_data
{
int fd;
time_t begin;
};
#define MAX_64BIT_STRLEN 20
static void
set_attribute(xmlNodePtr node, const char *name, int value)
{
xmlChar scratch[MAX_64BIT_STRLEN + 1] = {};
xmlStrPrintf(scratch, sizeof(scratch), BAD_CAST "%d", value);
xmlSetProp(node, BAD_CAST name, scratch);
}
/**
* Output the given event.
*
* @param parent the parent node to add new content to.
* @param event the event to write out.
*/
static void
emit_event(xmlNodePtr parent, struct zuc_event *event)
{
char *msg = NULL;
switch (event->op) {
case ZUC_OP_TRUE:
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: false\n"
"Expected: true\n", event->file, event->line,
event->expr1) < 0) {
msg = NULL;
}
break;
case ZUC_OP_FALSE:
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: true\n"
"Expected: false\n", event->file, event->line,
event->expr1) < 0) {
msg = NULL;
}
break;
case ZUC_OP_NULL:
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: %p\n"
"Expected: %p\n", event->file, event->line,
event->expr1, (void *)event->val1, NULL) < 0) {
msg = NULL;
}
break;
case ZUC_OP_NOT_NULL:
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: %p\n"
"Expected: not %p\n", event->file, event->line,
event->expr1, (void *)event->val1, NULL) < 0) {
msg = NULL;
}
break;
case ZUC_OP_EQ:
if (event->valtype == ZUC_VAL_CSTR) {
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: %s\n"
"Expected: %s\n"
"Which is: %s\n",
event->file, event->line, event->expr2,
(char *)event->val2, event->expr1,
(char *)event->val1) < 0) {
msg = NULL;
}
} else {
if (asprintf(&msg, "%s:%d: error: Value of: %s\n"
" Actual: %ld\n"
"Expected: %s\n"
"Which is: %ld\n",
event->file, event->line, event->expr2,
event->val2, event->expr1,
event->val1) < 0) {
msg = NULL;
}
}
break;
case ZUC_OP_NE:
if (event->valtype == ZUC_VAL_CSTR) {
if (asprintf(&msg, "%s:%d: error: "
"Expected: (%s) %s (%s),"
" actual: %s == %s\n",
event->file, event->line,
event->expr1, zuc_get_opstr(event->op),
event->expr2, (char *)event->val1,
(char *)event->val2) < 0) {
msg = NULL;
}
} else {
if (asprintf(&msg, "%s:%d: error: "
"Expected: (%s) %s (%s),"
" actual: %ld vs %ld\n",
event->file, event->line,
event->expr1, zuc_get_opstr(event->op),
event->expr2, event->val1,
event->val2) < 0) {
msg = NULL;
}
}
break;
case ZUC_OP_TERMINATE:
{
char const *level = (event->val1 == 0) ? "error"
: (event->val1 == 1) ? "warning"
: "note";
if (asprintf(&msg, "%s:%d: %s: %s\n",
event->file, event->line, level,
event->expr1) < 0) {
msg = NULL;
}
break;
}
case ZUC_OP_TRACEPOINT:
if (asprintf(&msg, "%s:%d: note: %s\n",
event->file, event->line, event->expr1) < 0) {
msg = NULL;
}
break;
default:
if (asprintf(&msg, "%s:%d: error: "
"Expected: (%s) %s (%s), actual: %ld vs %ld\n",
event->file, event->line,
event->expr1, zuc_get_opstr(event->op),
event->expr2, event->val1, event->val2) < 0) {
msg = NULL;
}
}
if ((event->op == ZUC_OP_TERMINATE) && (event->val1 > 1)) {
xmlNewChild(parent, NULL, BAD_CAST "skipped", NULL);
} else {
xmlNodePtr node = xmlNewChild(parent, NULL,
BAD_CAST "failure", NULL);
if (msg) {
xmlSetProp(node, BAD_CAST "message", BAD_CAST msg);
}
xmlSetProp(node, BAD_CAST "type", BAD_CAST "");
if (msg) {
xmlNodePtr cdata = xmlNewCDataBlock(node->doc,
BAD_CAST msg,
strlen(msg));
xmlAddChild(node, cdata);
}
}
free(msg);
}
/**
* Formats a time in milliseconds to the normal JUnit elapsed form, or
* NULL if there is a problem.
* The caller should release this with free()
*
* @return the formatted time string upon success, NULL otherwise.
*/
static char *
as_duration(long ms) {
char *str = NULL;
if (asprintf(&str, "%1.3f", ms / 1000.0) < 0) {
str = NULL;
} else {
/*
* Special case to match behavior of standard JUnit output
* writers. Asumption is certain readers might have
* limitations, etc. so it is best to keep 100% identical
* output.
*/
if (!strcmp("0.000", str)) {
free(str);
str = strdup("0");
}
}
return str;
}
/**
* Returns the status string for the tests (run/notrun).
*
* @param test the test to check status of.
* @return the status string.
*/
static char const *
get_test_status(struct zuc_test *test)
{
if (test->disabled || test->skipped)
return "notrun";
else
return "run";
}
/**
* Output the given test.
*
* @param parent the parent node to add new content to.
* @param test the test to write out.
*/
static void
emit_test(xmlNodePtr parent, struct zuc_test *test)
{
char *time_str = as_duration(test->elapsed);
xmlNodePtr node = xmlNewChild(parent, NULL, BAD_CAST "testcase", NULL);
xmlSetProp(node, BAD_CAST "name", BAD_CAST test->name);
xmlSetProp(node, BAD_CAST "status", BAD_CAST get_test_status(test));
if (time_str) {
xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str);
free(time_str);
time_str = NULL;
}
xmlSetProp(node, BAD_CAST "classname", BAD_CAST test->test_case->name);
if ((test->failed || test->fatal || test->skipped) && test->events) {
struct zuc_event *evt;
for (evt = test->events; evt; evt = evt->next)
emit_event(node, evt);
}
}
/**
* Output the given test case.
*
* @param parent the parent node to add new content to.
* @param test_case the test case to write out.
*/
static void
emit_case(xmlNodePtr parent, struct zuc_case *test_case)
{
int i;
int skipped = 0;
int disabled = 0;
int failures = 0;
xmlNodePtr node = NULL;
char *time_str = as_duration(test_case->elapsed);
for (i = 0; i < test_case->test_count; ++i) {
if (test_case->tests[i]->disabled )
disabled++;
if (test_case->tests[i]->skipped )
skipped++;
if (test_case->tests[i]->failed
|| test_case->tests[i]->fatal )
failures++;
}
node = xmlNewChild(parent, NULL, BAD_CAST "testsuite", NULL);
xmlSetProp(node, BAD_CAST "name", BAD_CAST test_case->name);
set_attribute(node, "tests", test_case->test_count);
set_attribute(node, "failures", failures);
set_attribute(node, "disabled", disabled);
set_attribute(node, "skipped", skipped);
if (time_str) {
xmlSetProp(node, BAD_CAST "time", BAD_CAST time_str);
free(time_str);
time_str = NULL;
}
for (i = 0; i < test_case->test_count; ++i)
emit_test(node, test_case->tests[i]);
}
/**
* Formats a time in milliseconds to the full ISO-8601 date/time string
* format, or NULL if there is a problem.
* The caller should release this with free()
*
* @return the formatted time string upon success, NULL otherwise.
*/
static char *
as_iso_8601(time_t const *t)
{
char *result = NULL;
char buf[32] = {};
struct tm when;
if (gmtime_r(t, &when) != NULL)
if (strftime(buf, sizeof(buf), ISO_8601_FORMAT, &when))
result = strdup(buf);
return result;
}
static void
run_started(void *data, int live_case_count, int live_test_count,
int disabled_count)
{
struct junit_data *jdata = data;
jdata->begin = time(NULL);
jdata->fd = open(XML_FNAME, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
}
static void
run_ended(void *data, int case_count, struct zuc_case **cases,
int live_case_count, int live_test_count, int total_passed,
int total_failed, int total_disabled, long total_elapsed)
{
int i;
long time = 0;
char *time_str = NULL;
char *timestamp = NULL;
xmlNodePtr root = NULL;
xmlDocPtr doc = NULL;
xmlChar *xmlchars = NULL;
int xmlsize = 0;
struct junit_data *jdata = data;
for (i = 0; i < case_count; ++i)
time += cases[i]->elapsed;
time_str = as_duration(time);
timestamp = as_iso_8601(&jdata->begin);
/* here would be where to add errors? */
doc = xmlNewDoc(BAD_CAST "1.0");
root = xmlNewNode(NULL, BAD_CAST "testsuites");
xmlDocSetRootElement(doc, root);
set_attribute(root, "tests", live_test_count);
set_attribute(root, "failures", total_failed);
set_attribute(root, "disabled", total_disabled);
if (timestamp) {
xmlSetProp(root, BAD_CAST "timestamp", BAD_CAST timestamp);
free(timestamp);
timestamp = NULL;
}
if (time_str) {
xmlSetProp(root, BAD_CAST "time", BAD_CAST time_str);
free(time_str);
time_str = NULL;
}
xmlSetProp(root, BAD_CAST "name", BAD_CAST "AllTests");
for (i = 0; i < case_count; ++i) {
emit_case(root, cases[i]);
}
xmlDocDumpFormatMemoryEnc(doc, &xmlchars, &xmlsize, "UTF-8", 1);
dprintf(jdata->fd, "%s", (char *) xmlchars);
xmlFree(xmlchars);
xmlchars = NULL;
xmlFreeDoc(doc);
if ((jdata->fd != fileno(stdout))
&& (jdata->fd != fileno(stderr))
&& (jdata->fd != -1)) {
close(jdata->fd);
jdata->fd = -1;
}
}
static void
destroy(void *data)
{
xmlCleanupParser();
free(data);
}
struct zuc_event_listener *
zuc_junit_reporter_create(void)
{
struct zuc_event_listener *listener =
zalloc(sizeof(struct zuc_event_listener));
struct junit_data *data = zalloc(sizeof(struct junit_data));
data->fd = -1;
listener->data = data;
listener->destroy = destroy;
listener->run_started = run_started;
listener->run_ended = run_ended;
return listener;
}
#else /* ENABLE_JUNIT_XML */
#include "shared/zalloc.h"
#include "zuc_event_listener.h"
/*
* Simple stub version if junit output (including libxml2 support) has
* been disabled.
* Will return NULL to cause failures as calling this when the #define
* has not been enabled is an invalid scenario.
*/
struct zuc_event_listener *
zuc_junit_reporter_create(void)
{
return NULL;
}
#endif /* ENABLE_JUNIT_XML */

@ -0,0 +1,38 @@
/*
* Copyright © 2015 Samsung Electronics Co., Ltd
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef ZUC_JUNIT_REPORTER_H
#define ZUC_JUNIT_REPORTER_H
struct zuc_event_listener;
/**
* Creates an instance of a reporter that will write data in the JUnit
* XML format.
*/
struct zuc_event_listener *
zuc_junit_reporter_create(void);
#endif /* ZUC_JUNIT_REPORTER_H */

@ -44,6 +44,7 @@
#include "zuc_collector.h"
#include "zuc_context.h"
#include "zuc_event_listener.h"
#include "zuc_junit_reporter.h"
#include "shared/config-parser.h"
#include "shared/helpers.h"
@ -152,6 +153,12 @@ zuc_set_break_on_failure(bool break_on_failure)
g_ctx.break_on_failure = break_on_failure;
}
void
zuc_set_output_junit(bool enable)
{
g_ctx.output_junit = enable;
}
const char *
zuc_get_program_name(void)
{
@ -523,6 +530,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
int opt_repeat = 0;
int opt_random = 0;
int opt_break_on_failure = 0;
int opt_junit = 0;
char *opt_filter = NULL;
char *help_param = NULL;
@ -535,6 +543,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
{ WESTON_OPTION_INTEGER, "zuc-random", 0, &opt_random },
{ WESTON_OPTION_BOOLEAN, "zuc-break-on-failure", 0,
&opt_break_on_failure },
#if ENABLE_JUNIT_XML
{ WESTON_OPTION_BOOLEAN, "zuc-output-xml", 0, &opt_junit },
#endif
{ WESTON_OPTION_STRING, "zuc-filter", 0, &opt_filter },
};
@ -623,6 +634,9 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
" --zuc-filter=FILTER\n"
" --zuc-list-tests\n"
" --zuc-nofork\n"
#if ENABLE_JUNIT_XML
" --zuc-output-xml\n"
#endif
" --zuc-random=N [0|1|<seed number>]\n"
" --zuc-repeat=N\n"
" --help\n",
@ -638,6 +652,7 @@ zuc_initialize(int *argc, char *argv[], bool *help_flagged)
zuc_set_random(opt_random);
zuc_set_spawn(!opt_nofork);
zuc_set_break_on_failure(opt_break_on_failure);
zuc_set_output_junit(opt_junit);
rc = EXIT_SUCCESS;
}
@ -1301,6 +1316,8 @@ zucimpl_run_tests(void)
if (g_ctx.listeners == NULL) {
zuc_add_event_listener(zuc_collector_create(&(g_ctx.fds[1])));
zuc_add_event_listener(zuc_base_logger_create());
if (g_ctx.output_junit)
zuc_add_event_listener(zuc_junit_reporter_create());
}
if (g_ctx.case_count < 1) {

Loading…
Cancel
Save