From 11798eeb34bcc367dada3baa98734d737ec9ee63 Mon Sep 17 00:00:00 2001 From: Robin Jonsson Date: Thu, 2 Aug 2018 11:31:55 +0200 Subject: [PATCH 1/8] Allow custom event body serializer for HEC Customizable & pluggable event serializer support for the HttpEventCollectorAdapter. --- README.md | 2 + .../splunk/logging/EventBodySerializer.java | 53 +++++++++++++++++++ .../HttpEventCollectorLog4jAppender.java | 16 ++++-- .../HttpEventCollectorLogbackAppender.java | 19 ++++++- .../HttpEventCollectorLoggingHandler.java | 9 +++- .../logging/HttpEventCollectorSender.java | 42 +++++---------- .../com/splunk/logging/MessageFormat.java | 1 - src/test/java/HttpEventCollectorUnitTest.java | 5 +- src/test/resources/log4j2_template.xml | 1 + src/test/resources/logback.xml | 2 +- src/test/resources/logback_template.xml | 1 + 11 files changed, 114 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/splunk/logging/EventBodySerializer.java diff --git a/README.md b/README.md index d8e3d7a4..d441f572 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ LOGGER.info("hello world"); #### Message Format An event message format could be configured for HTTP event appender in logging framework configuration. It could have one of the two possible values - text, json. It is an optional property with default value as 'text'. Message format 'json' is used where the event message could be in json format. +It is also possible to use a custom event body serializer for the HTTP event adapter, to format the logging event however you please. Simply create a class implementing `com.splunk.logging.EventBodySerializer`, and add the full class name as a property (`eventBodySerializer`) to the adapter. Default will be a JSON event body containing message, severity, and other properties. + For more information, see http://dev.splunk.com/view/SP-CAAAE2K. # License diff --git a/src/main/java/com/splunk/logging/EventBodySerializer.java b/src/main/java/com/splunk/logging/EventBodySerializer.java new file mode 100644 index 00000000..139e2b88 --- /dev/null +++ b/src/main/java/com/splunk/logging/EventBodySerializer.java @@ -0,0 +1,53 @@ +package com.splunk.logging; + +import java.io.Serializable; +import java.util.Map; +import org.json.simple.JSONObject; + +public interface EventBodySerializer { + + String serializeEventBody( + HttpEventCollectorEventInfo eventInfo, + String formattedMessage + ); + + class Default implements EventBodySerializer { + + @Override + public String serializeEventBody( + final HttpEventCollectorEventInfo eventInfo, + final String formattedMessage + ) { + final JSONObject body = new JSONObject(); + putIfPresent(body, "severity", eventInfo.getSeverity()); + putIfPresent(body, "message", formattedMessage); + putIfPresent(body, "logger", eventInfo.getLoggerName()); + putIfPresent(body, "thread", eventInfo.getThreadName()); + // add an exception record if and only if there is one + // in practice, the message also has the exception information attached + if (eventInfo.getExceptionMessage() != null) { + putIfPresent(body, "exception", eventInfo.getExceptionMessage()); + } + + // add properties if and only if there are any + final Map props = eventInfo.getProperties(); + if (props != null && !props.isEmpty()) { + body.put("properties", props); + } + // add marker if and only if there is one + final Serializable marker = eventInfo.getMarker(); + if (marker != null) { + putIfPresent(body, "marker", marker.toString()); + } + + return body.toString(); + } + + private void putIfPresent(final JSONObject obj, String tag, Object value) { + if (value != null && value instanceof String && ((String) value).isEmpty()) { + return; + } + obj.put(tag, value); + } + } +} diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java b/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java index aa58591d..125c9697 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorLog4jAppender.java @@ -68,7 +68,8 @@ private HttpEventCollectorLog4jAppender(final String name, long retriesOnError, String sendMode, String middleware, - final String disableCertificateValidation) + final String disableCertificateValidation, + final String eventBodySerializer) { super(name, filter, layout, ignoreExceptions); Dictionary metadata = new Hashtable(); @@ -84,7 +85,13 @@ private HttpEventCollectorLog4jAppender(final String name, if (middleware != null && !middleware.isEmpty()) { try { this.sender.addMiddleware((HttpEventCollectorMiddleware.HttpSenderMiddleware)(Class.forName(middleware).newInstance())); - } catch (Exception e) {} + } catch (Exception ignored) {} + } + + if (eventBodySerializer != null && !eventBodySerializer.isEmpty()) { + try { + this.sender.setEventBodySerializer((EventBodySerializer) Class.forName(eventBodySerializer).newInstance()); + } catch (final Exception ignored) {} } // plug resend middleware @@ -128,6 +135,7 @@ public static HttpEventCollectorLog4jAppender createAppender( @PluginAttribute("send_mode") final String sendMode, @PluginAttribute("middleware") final String middleware, @PluginAttribute("disableCertificateValidation") final String disableCertificateValidation, + @PluginAttribute("eventBodySerializer") final String eventBodySerializer, @PluginAttribute(value = "includeLoggerName", defaultBoolean = true) final boolean includeLoggerName, @PluginAttribute(value = "includeThreadName", defaultBoolean = true) final boolean includeThreadName, @PluginAttribute(value = "includeMDC", defaultBoolean = true) final boolean includeMDC, @@ -174,7 +182,9 @@ public static HttpEventCollectorLog4jAppender createAppender( parseInt(retriesOnError, 0), sendMode, middleware, - disableCertificateValidation); + disableCertificateValidation, + eventBodySerializer + ); } diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java b/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java index 29a3538b..9ac376bf 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorLogbackAppender.java @@ -46,6 +46,7 @@ public class HttpEventCollectorLogbackAppender extends AppenderBase { private String _type; private String _disableCertificateValidation; private String _middleware; + private String _eventBodySerializer; private long _batchInterval = 0; private long _batchCount = 0; private long _batchSize = 0; @@ -81,7 +82,13 @@ public void start() { if (_middleware != null && !_middleware.isEmpty()) { try { this.sender.addMiddleware((HttpEventCollectorMiddleware.HttpSenderMiddleware)(Class.forName(_middleware).newInstance())); - } catch (Exception e) {} + } catch (Exception ignored) {} + } + + if (_eventBodySerializer != null && !_eventBodySerializer.isEmpty()) { + try { + this.sender.setEventBodySerializer((EventBodySerializer) Class.forName(_eventBodySerializer).newInstance()); + } catch (final Exception ignored) {} } // plug resend middleware @@ -120,7 +127,7 @@ private void sendEvent(ILoggingEvent event) { } MarkerConverter c = new MarkerConverter(); - if (event != null && started) { + if (this.started) { this.sender.send( event.getLevel().toString(), _layout.doLayout((E) event), @@ -257,6 +264,10 @@ public String getIndex() { return this._index; } + public String getEventBodySerializer() { + return _eventBodySerializer; + } + public void setDisableCertificateValidation(String disableCertificateValidation) { this._disableCertificateValidation = disableCertificateValidation; } @@ -289,6 +300,10 @@ public String getDisableCertificateValidation() { return _disableCertificateValidation; } + public void setEventBodySerializer(String eventBodySerializer) { + this._eventBodySerializer = eventBodySerializer; + } + private static long parseLong(String string, int defaultValue) { try { return Long.parseLong(string); diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java b/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java index ab0ded57..fd96d32f 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java @@ -149,6 +149,7 @@ public HttpEventCollectorLoggingHandler() { long retriesOnError = getConfigurationNumericProperty(RetriesOnErrorTag, 0); String sendMode = getConfigurationProperty(SendModeTag, "sequential"); String middleware = getConfigurationProperty(MiddlewareTag, ""); + String eventBodySerializer = getConfigurationProperty("eventBodySerializer", ""); includeLoggerName = getConfigurationBooleanProperty(IncludeLoggerNameConfTag, true); includeThreadName = getConfigurationBooleanProperty(IncludeThreadNameConfTag, true); @@ -162,7 +163,13 @@ public HttpEventCollectorLoggingHandler() { if (middleware != null && !middleware.isEmpty()) { try { this.sender.addMiddleware((HttpEventCollectorMiddleware.HttpSenderMiddleware)(Class.forName(middleware).newInstance())); - } catch (Exception e) {} + } catch (Exception ignored) {} + } + + if (eventBodySerializer != null && !eventBodySerializer.isEmpty()) { + try { + this.sender.setEventBodySerializer((EventBodySerializer) Class.forName(eventBodySerializer).newInstance()); + } catch (final Exception ignored) {} } // plug retries middleware diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java index 667f8e59..39052e7a 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java @@ -50,7 +50,7 @@ /** * This is an internal helper class that sends logging events to Splunk http event collector. */ -final class HttpEventCollectorSender extends TimerTask implements HttpEventCollectorMiddleware.IHttpSender { +public class HttpEventCollectorSender extends TimerTask implements HttpEventCollectorMiddleware.IHttpSender { public static final String MetadataTimeTag = "time"; public static final String MetadataHostTag = "host"; public static final String MetadataIndexTag = "index"; @@ -99,6 +99,7 @@ public enum SendMode private SendMode sendMode = SendMode.Sequential; private HttpEventCollectorMiddleware middleware = new HttpEventCollectorMiddleware(); private final MessageFormat messageFormat; + private EventBodySerializer eventBodySerializer; /** * Initialize HttpEventCollectorSender @@ -135,12 +136,12 @@ public HttpEventCollectorSender( this.maxEventsBatchCount = maxEventsBatchCount; this.maxEventsBatchSize = maxEventsBatchSize; this.metadata = metadata; - + final String format = metadata.get(MetadataMessageFormatTag); // Get MessageFormat enum from format string. Do this once per instance in constructor to avoid expensive operation in // each event sender call this.messageFormat = MessageFormat.fromFormat(format); - + if (sendModeStr != null) { if (sendModeStr.equals(SendModeSequential)) this.sendMode = SendMode.Sequential; @@ -239,6 +240,10 @@ public void disableCertificateValidation() { disableCertificateValidation = true; } + public void setEventBodySerializer(EventBodySerializer eventBodySerializer) { + this.eventBodySerializer = eventBodySerializer; + } + @SuppressWarnings("unchecked") private static void putIfPresent(JSONObject collection, String tag, Object value) { if (value != null) { @@ -263,34 +268,15 @@ private String serializeEventInfo(HttpEventCollectorEventInfo eventInfo) { putIfPresent(event, MetadataIndexTag, metadata.get(MetadataIndexTag)); putIfPresent(event, MetadataSourceTag, metadata.get(MetadataSourceTag)); putIfPresent(event, MetadataSourceTypeTag, metadata.get(MetadataSourceTypeTag)); - + // Parse message on the basis of format - final Object parsedMessage = this.messageFormat.parse(eventInfo.getMessage()); - - // event body - JSONObject body = new JSONObject(); - putIfPresent(body, "severity", eventInfo.getSeverity()); - putIfPresent(body, "message", parsedMessage); - putIfPresent(body, "logger", eventInfo.getLoggerName()); - putIfPresent(body, "thread", eventInfo.getThreadName()); - // add an exception record if and only if there is one - // in practice, the message also has the exception information attached - if (eventInfo.getExceptionMessage() != null) { - putIfPresent(body, "exception", eventInfo.getExceptionMessage()); - } + final String parsedMessage = this.messageFormat.parse(eventInfo.getMessage()).toString(); - // add properties if and only if there are any - final Map props = eventInfo.getProperties(); - if (props != null && !props.isEmpty()) { - body.put("properties", props); + if (eventBodySerializer == null) { + eventBodySerializer = new EventBodySerializer.Default(); } - // add marker if and only if there is one - final Serializable marker = eventInfo.getMarker(); - if (marker != null) { - putIfPresent(body, "marker", marker.toString()); - } - // join event and body - event.put("event", body); + + event.put("event", eventBodySerializer.serializeEventBody(eventInfo, parsedMessage)); return event.toString(); } diff --git a/src/main/java/com/splunk/logging/MessageFormat.java b/src/main/java/com/splunk/logging/MessageFormat.java index 4ff81a73..d27add07 100644 --- a/src/main/java/com/splunk/logging/MessageFormat.java +++ b/src/main/java/com/splunk/logging/MessageFormat.java @@ -30,7 +30,6 @@ enum MessageFormat { * * * @param message the message string - * @param format the message format * * @return parsed message object based on format */ diff --git a/src/test/java/HttpEventCollectorUnitTest.java b/src/test/java/HttpEventCollectorUnitTest.java index 99f9046c..4bdb0827 100644 --- a/src/test/java/HttpEventCollectorUnitTest.java +++ b/src/test/java/HttpEventCollectorUnitTest.java @@ -43,6 +43,7 @@ public void log4j_simple() throws Exception { userInputs.put("user_middleware", "HttpEventCollectorUnitTestMiddleware"); userInputs.put("user_batch_size_count", "1"); userInputs.put("user_batch_size_bytes", "0"); + userInputs.put("user_eventBodySerializer", "DoesNotExistButShouldNotCrashTest"); TestUtil.resetLog4j2Configuration("log4j2_template.xml", "log4j2.xml", userInputs); org.apache.logging.log4j.Logger LOG4J = org.apache.logging.log4j.LogManager.getLogger(loggerName); @@ -71,6 +72,7 @@ public void logback_simple() throws Exception { userInputs.put("user_logger_name", loggerName); userInputs.put("user_httpEventCollector_token", "11111111-2222-3333-4444-555555555555"); userInputs.put("user_middleware", "HttpEventCollectorUnitTestMiddleware"); + userInputs.put("user_eventBodySerializer", "DoesNotExistButShouldNotCrashTest"); TestUtil.resetLogbackConfiguration("logback_template.xml", "logback.xml", userInputs); org.slf4j.Logger LOGBACK = org.slf4j.LoggerFactory.getLogger(loggerName); @@ -101,7 +103,8 @@ public void java_util_logger_simple() { "com.splunk.logging.HttpEventCollectorLoggingHandler.batch_size_count=0\n" + "com.splunk.logging.HttpEventCollectorLoggingHandler.batch_size_bytes=0\n" + "com.splunk.logging.HttpEventCollectorLoggingHandler.batch_interval=0\n" + - "com.splunk.logging.HttpEventCollectorLoggingHandler.middleware=HttpEventCollectorUnitTestMiddleware\n" + "com.splunk.logging.HttpEventCollectorLoggingHandler.middleware=HttpEventCollectorUnitTestMiddleware\n" + + "com.splunk.logging.HttpEventCollectorLoggingHandler.eventBodySerializer=DoesNotExistButShouldNotCrashTest\n" ); // send 3 events diff --git a/src/test/resources/log4j2_template.xml b/src/test/resources/log4j2_template.xml index 82d5b8af..a1d288ac 100644 --- a/src/test/resources/log4j2_template.xml +++ b/src/test/resources/log4j2_template.xml @@ -16,6 +16,7 @@ disableCertificateValidation="true" send_mode="%user_send_mode%" middleware="%user_middleware%" + eventBodySerializer="%user_eventBodySerializer%" > diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index 1ea8d865..2a66042b 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -58,7 +58,7 @@ under the License. splunktest battlecat text - HttpEventCollectorUnitTestMiddleware + HttpEventCollectorUnitTestMiddleware %msg diff --git a/src/test/resources/logback_template.xml b/src/test/resources/logback_template.xml index 339bc97b..e0585d87 100644 --- a/src/test/resources/logback_template.xml +++ b/src/test/resources/logback_template.xml @@ -17,6 +17,7 @@ %user_send_mode% %user_send_mode% %user_middleware% + %user_eventBodySerializer% %msg From 6d12f035defc4d26c0d917c210b3da0b6d3c2bc1 Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 10:31:36 -0700 Subject: [PATCH 2/8] fix the event body serilizer to correctly parse json message sent to splunk and make sure tests passing; fix some typo --- .../splunk/logging/EventBodySerializer.java | 24 +++++++++++++++---- .../logging/HttpEventCollectorSender.java | 4 ++-- .../HttpEventCollector_JavaLoggingTest.java | 2 +- .../java/HttpEventCollector_LogbackTest.java | 2 +- src/test/java/TestUtil.java | 20 +++++++++------- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/splunk/logging/EventBodySerializer.java b/src/main/java/com/splunk/logging/EventBodySerializer.java index 139e2b88..029541b7 100644 --- a/src/main/java/com/splunk/logging/EventBodySerializer.java +++ b/src/main/java/com/splunk/logging/EventBodySerializer.java @@ -8,15 +8,20 @@ public interface EventBodySerializer { String serializeEventBody( HttpEventCollectorEventInfo eventInfo, - String formattedMessage + Object formattedMessage + ); + + JSONObject serializeEventBodyAsJson( + HttpEventCollectorEventInfo eventInfo, + Object formattedMessage ); class Default implements EventBodySerializer { @Override - public String serializeEventBody( - final HttpEventCollectorEventInfo eventInfo, - final String formattedMessage + public JSONObject serializeEventBodyAsJson( + final HttpEventCollectorEventInfo eventInfo, + final Object formattedMessage ) { final JSONObject body = new JSONObject(); putIfPresent(body, "severity", eventInfo.getSeverity()); @@ -40,7 +45,16 @@ public String serializeEventBody( putIfPresent(body, "marker", marker.toString()); } - return body.toString(); + return body; + } + + @Override + public String serializeEventBody( + final HttpEventCollectorEventInfo eventInfo, + final Object formattedMessage + ) { + + return this.serializeEventBodyAsJson(eventInfo,formattedMessage).toString(); } private void putIfPresent(final JSONObject obj, String tag, Object value) { diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java index 39052e7a..1b2dde4a 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java @@ -270,13 +270,13 @@ private String serializeEventInfo(HttpEventCollectorEventInfo eventInfo) { putIfPresent(event, MetadataSourceTypeTag, metadata.get(MetadataSourceTypeTag)); // Parse message on the basis of format - final String parsedMessage = this.messageFormat.parse(eventInfo.getMessage()).toString(); + final Object parsedMessage = this.messageFormat.parse(eventInfo.getMessage()); if (eventBodySerializer == null) { eventBodySerializer = new EventBodySerializer.Default(); } - event.put("event", eventBodySerializer.serializeEventBody(eventInfo, parsedMessage)); + event.put("event", eventBodySerializer.serializeEventBodyAsJson(eventInfo, parsedMessage)); return event.toString(); } diff --git a/src/test/java/HttpEventCollector_JavaLoggingTest.java b/src/test/java/HttpEventCollector_JavaLoggingTest.java index 3c627ceb..2705247a 100644 --- a/src/test/java/HttpEventCollector_JavaLoggingTest.java +++ b/src/test/java/HttpEventCollector_JavaLoggingTest.java @@ -464,7 +464,7 @@ private void canSendJsonEventUsingUtilLoggerWithSourceType(final String sourceTy final JSONObject jsonObject = new JSONObject(); jsonObject.put("transactionId", "11"); jsonObject.put("userId", "21"); - jsonObject.put("eventTimestap", timeMillsec); + jsonObject.put("eventTimestamp", timeMillsec); final Logger logger = Logger.getLogger(loggerName); diff --git a/src/test/java/HttpEventCollector_LogbackTest.java b/src/test/java/HttpEventCollector_LogbackTest.java index 68e8f043..4f7b0367 100644 --- a/src/test/java/HttpEventCollector_LogbackTest.java +++ b/src/test/java/HttpEventCollector_LogbackTest.java @@ -396,7 +396,7 @@ private void canSendJsonEventUsingLogbackWithSourceType(final String sourceType) final JSONObject jsonObject = new JSONObject(); jsonObject.put("transactionId", "11"); jsonObject.put("userId", "21"); - jsonObject.put("eventTimestap", timeMillsec); + jsonObject.put("eventTimestamp", timeMillsec); final Logger logger = LoggerFactory.getLogger(loggerName); diff --git a/src/test/java/TestUtil.java b/src/test/java/TestUtil.java index c47400f8..631695d0 100644 --- a/src/test/java/TestUtil.java +++ b/src/test/java/TestUtil.java @@ -377,7 +377,7 @@ public static void verifyNoEventSentToSplunk(List msgs) throws IOExcepti /* verify each of the message in msgs appeared and appeared only once in splunk */ - public static void verifyEventsSentToSplunk(List msgs) throws IOException { + public static void verifyEventsSentToSplunk(List msgs) throws IOException, InterruptedException { connectToSplunk(); for (String msg : msgs) { @@ -392,7 +392,7 @@ public static void verifyEventsSentToSplunk(List msgs) throws IOExceptio } else { resultsStream = service.oneshotSearch("search " + msg); } - + resultsReader = new ResultsReaderXml(resultsStream); //verify has one and only one record return @@ -404,6 +404,8 @@ public static void verifyEventsSentToSplunk(List msgs) throws IOExceptio if (eventCount > 0) break; + + Thread.sleep(5000); } resultsReader.close(); @@ -425,14 +427,14 @@ private static InputStream searchJsonMessageEvent(final JSONObject jsonObject) { boolean firstSearchTerm = true; for (final Object entryObject : jsonObject.entrySet()) { final Entry jsonEntry = (Entry) entryObject; - if (firstSearchTerm) { - searchQuery += String.format("search \"message.%s\"=%s", jsonEntry.getKey(), jsonEntry.getValue()); - firstSearchTerm = false; - } else { - searchQuery += String.format(" | search \"message.%s\"=%s", jsonEntry.getKey(), jsonEntry.getValue()); - } + if (firstSearchTerm) { + searchQuery += String.format("search \"message.%s\"=%s", jsonEntry.getKey(), jsonEntry.getValue()); + firstSearchTerm = false; + } else { + searchQuery += String.format(" | search \"message.%s\"=%s", jsonEntry.getKey(), jsonEntry.getValue()); + } } - + return service.oneshotSearch(searchQuery); } From 273a387d019eb8df2ba7c8b6fced3194c135d9fd Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 10:40:40 -0700 Subject: [PATCH 3/8] simplify the serilizer interface --- .../splunk/logging/EventBodySerializer.java | 18 ++---------------- .../logging/HttpEventCollectorSender.java | 2 +- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/splunk/logging/EventBodySerializer.java b/src/main/java/com/splunk/logging/EventBodySerializer.java index 029541b7..a048b0fd 100644 --- a/src/main/java/com/splunk/logging/EventBodySerializer.java +++ b/src/main/java/com/splunk/logging/EventBodySerializer.java @@ -11,15 +11,10 @@ String serializeEventBody( Object formattedMessage ); - JSONObject serializeEventBodyAsJson( - HttpEventCollectorEventInfo eventInfo, - Object formattedMessage - ); - class Default implements EventBodySerializer { @Override - public JSONObject serializeEventBodyAsJson( + public String serializeEventBody( final HttpEventCollectorEventInfo eventInfo, final Object formattedMessage ) { @@ -45,16 +40,7 @@ public JSONObject serializeEventBodyAsJson( putIfPresent(body, "marker", marker.toString()); } - return body; - } - - @Override - public String serializeEventBody( - final HttpEventCollectorEventInfo eventInfo, - final Object formattedMessage - ) { - - return this.serializeEventBodyAsJson(eventInfo,formattedMessage).toString(); + return body.toString(); } private void putIfPresent(final JSONObject obj, String tag, Object value) { diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java index 1b2dde4a..f9fc626f 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java @@ -276,7 +276,7 @@ private String serializeEventInfo(HttpEventCollectorEventInfo eventInfo) { eventBodySerializer = new EventBodySerializer.Default(); } - event.put("event", eventBodySerializer.serializeEventBodyAsJson(eventInfo, parsedMessage)); + event.put("event", eventBodySerializer.serializeEventBody(eventInfo, parsedMessage)); return event.toString(); } From db47b2fcdeaf9a5e67ddd9d749473a9ceff93384 Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 10:47:48 -0700 Subject: [PATCH 4/8] code simplify: remove redundant helper function --- .../splunk/logging/EventBodySerializer.java | 25 +++++++------------ .../logging/HttpEventCollectorSender.java | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/splunk/logging/EventBodySerializer.java b/src/main/java/com/splunk/logging/EventBodySerializer.java index a048b0fd..1957d5e4 100644 --- a/src/main/java/com/splunk/logging/EventBodySerializer.java +++ b/src/main/java/com/splunk/logging/EventBodySerializer.java @@ -7,8 +7,8 @@ public interface EventBodySerializer { String serializeEventBody( - HttpEventCollectorEventInfo eventInfo, - Object formattedMessage + HttpEventCollectorEventInfo eventInfo, + Object formattedMessage ); class Default implements EventBodySerializer { @@ -19,35 +19,28 @@ public String serializeEventBody( final Object formattedMessage ) { final JSONObject body = new JSONObject(); - putIfPresent(body, "severity", eventInfo.getSeverity()); - putIfPresent(body, "message", formattedMessage); - putIfPresent(body, "logger", eventInfo.getLoggerName()); - putIfPresent(body, "thread", eventInfo.getThreadName()); + HttpEventCollectorSender.putIfPresent(body, "severity", eventInfo.getSeverity()); + HttpEventCollectorSender.putIfPresent(body, "message", formattedMessage); + HttpEventCollectorSender.putIfPresent(body, "logger", eventInfo.getLoggerName()); + HttpEventCollectorSender.putIfPresent(body, "thread", eventInfo.getThreadName()); // add an exception record if and only if there is one // in practice, the message also has the exception information attached if (eventInfo.getExceptionMessage() != null) { - putIfPresent(body, "exception", eventInfo.getExceptionMessage()); + HttpEventCollectorSender.putIfPresent(body, "exception", eventInfo.getExceptionMessage()); } // add properties if and only if there are any - final Map props = eventInfo.getProperties(); + final Map props = eventInfo.getProperties(); if (props != null && !props.isEmpty()) { body.put("properties", props); } // add marker if and only if there is one final Serializable marker = eventInfo.getMarker(); if (marker != null) { - putIfPresent(body, "marker", marker.toString()); + HttpEventCollectorSender.putIfPresent(body, "marker", marker.toString()); } return body.toString(); } - - private void putIfPresent(final JSONObject obj, String tag, Object value) { - if (value != null && value instanceof String && ((String) value).isEmpty()) { - return; - } - obj.put(tag, value); - } } } diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java index f9fc626f..139b331a 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorSender.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorSender.java @@ -245,7 +245,7 @@ public void setEventBodySerializer(EventBodySerializer eventBodySerializer) { } @SuppressWarnings("unchecked") - private static void putIfPresent(JSONObject collection, String tag, Object value) { + public static void putIfPresent(JSONObject collection, String tag, Object value) { if (value != null) { if (value instanceof String && ((String) value).length() == 0) { // Do not add blank string From 1bba6b2455659884d89b30728fdb87acd45dd256 Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 11:29:46 -0700 Subject: [PATCH 5/8] output the serializer initilization error; add tests to show using user-defined eventBodySerializer --- .../HttpEventCollectorLoggingHandler.java | 5 +- .../HttpEventCollector_JavaLoggingTest.java | 50 +++++++++++++++---- src/test/java/testEventbodySerializer.java | 41 +++++++++++++++ .../resources/logging_template.properties | 2 + 4 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 src/test/java/testEventbodySerializer.java diff --git a/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java b/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java index fd96d32f..e1be4949 100644 --- a/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java +++ b/src/main/java/com/splunk/logging/HttpEventCollectorLoggingHandler.java @@ -169,7 +169,10 @@ public HttpEventCollectorLoggingHandler() { if (eventBodySerializer != null && !eventBodySerializer.isEmpty()) { try { this.sender.setEventBodySerializer((EventBodySerializer) Class.forName(eventBodySerializer).newInstance()); - } catch (final Exception ignored) {} + } catch (final Exception ex) { + //output error msg but not fail, it will default to use the default EventBodySerializer + System.out.println(ex); + } } // plug retries middleware diff --git a/src/test/java/HttpEventCollector_JavaLoggingTest.java b/src/test/java/HttpEventCollector_JavaLoggingTest.java index 2705247a..ff667921 100644 --- a/src/test/java/HttpEventCollector_JavaLoggingTest.java +++ b/src/test/java/HttpEventCollector_JavaLoggingTest.java @@ -16,9 +16,11 @@ import java.util.*; +import com.splunk.logging.EventBodySerializer; import com.splunk.logging.HttpEventCollectorErrorHandler; import com.splunk.logging.HttpEventCollectorEventInfo; +import com.splunk.logging.HttpEventCollectorSender; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -32,7 +34,7 @@ public final class HttpEventCollector_JavaLoggingTest { List logEx = new ArrayList(); /** - * sending a message via httplogging using log4j2 to splunk + * sending a message via httplogging using java.logging to splunk */ @Test public void canSendEventUsingJavaLogging() throws Exception { @@ -58,7 +60,7 @@ public void canSendEventUsingJavaLogging() throws Exception { } /** - * sending a message via httplogging using log4j2 to splunk + * sending a message via httplogging using java.logging to splunk */ @Test public void canSendEventUsingJavaLoggingWithOptions() throws Exception { @@ -86,7 +88,7 @@ public void canSendEventUsingJavaLoggingWithOptions() throws Exception { } /** - * sending batched message via httplogging to splunk + * sending batched message via http java.logging to splunk */ @Test public void sendBatchedEventsUsingJavaLogging() throws Exception { @@ -132,7 +134,7 @@ public void sendBatchedEventsByCount() throws Exception { loggerName = "splunkBatchLoggerCount"; userInputs.clear(); userInputs.put("user_httpEventCollector_token", token); - userInputs.put("user_batch_interval","0"); + userInputs.put("user_batch_interval", "0"); userInputs.put("user_batch_size_count", "5"); userInputs.put("user_logger_name", loggerName); userInputs.put("user_host", "host.example.com"); @@ -188,7 +190,7 @@ public void sendBatchedEventsByBatchsize() throws Exception { String loggerName = "splunkBatchLoggerSize"; HashMap userInputs = new HashMap(); userInputs.put("user_httpEventCollector_token", token); - userInputs.put("user_batch_interval","0"); + userInputs.put("user_batch_interval", "0"); userInputs.put("user_batch_size_bytes", "500"); userInputs.put("user_logger_name", loggerName); userInputs.put("user_host", "host.example.com"); @@ -332,7 +334,7 @@ public void error(final List data, final Exception break; Thread.sleep(1000); } - + // Enable httpEventCollector endpoint TestUtil.enableHttpEventCollector(); @@ -400,7 +402,7 @@ private String queueEvents() throws Exception { @Test public void eventsIsIndexedInOrderOfSent() throws Exception { TestUtil.enableHttpEventCollector(); - String indexName="httpevents_in_order_jl"; + String indexName = "httpevents_in_order_jl"; TestUtil.createIndex(indexName); String token = TestUtil.createHttpEventCollectorToken(httpEventCollectorName); @@ -418,9 +420,9 @@ public void eventsIsIndexedInOrderOfSent() throws Exception { List msgs = new ArrayList(); Date date = new Date(); int totalEventsCount = 1000; - String prefix="javalogging multiple events"; + String prefix = "javalogging multiple events"; for (int i = 0; i < totalEventsCount; i++) { - String jsonMsg = String.format("%s %s", prefix,i); + String jsonMsg = String.format("%s %s", prefix, i); logger.info(jsonMsg); msgs.add(jsonMsg); } @@ -447,6 +449,35 @@ public void canSendJsonEventUsingUtilLoggerWithDefaultSourceType() throws Except canSendJsonEventUsingUtilLoggerWithSourceType("battlecat_test"); } + /** + * sending a message using user-defined EventBodySerializer through java.logging to splunk + */ + @Test + public void canSendEventUsingJavaLoggingWithUserEventBodySerializer() throws Exception { + TestUtil.enableHttpEventCollector(); + + String token = TestUtil.createHttpEventCollectorToken(httpEventCollectorName); + + String loggerName = "splunkLoggerBodySerializer"; + HashMap userInputs = new HashMap(); + userInputs.put("user_httpEventCollector_token", token); + userInputs.put("user_logger_name", loggerName); + userInputs.put("user_eventBodySerializer", "testEventBodySerializer"); + + TestUtil.resetJavaLoggingConfiguration("logging_template.properties", "logging.properties", userInputs); + + Date date = new Date(); + String jsonMsg = String.format("EventDate:%s, EventMsg:test event for java logging With User EventBodySerializer", date.toString()); + + Logger logger = Logger.getLogger(loggerName); + logger.info(jsonMsg); + + TestUtil.verifyOneAndOnlyOneEventSentToSplunk("user-prefix:" + jsonMsg); + + TestUtil.deleteHttpEventCollectorToken(httpEventCollectorName); + } + + @SuppressWarnings("unchecked") private void canSendJsonEventUsingUtilLoggerWithSourceType(final String sourceType) throws Exception { final String token = TestUtil.createHttpEventCollectorToken(httpEventCollectorName); @@ -483,5 +514,4 @@ private void canSendJsonEventUsingUtilLoggerWithSourceType(final String sourceTy TestUtil.verifyEventsSentToSplunk(msgs); TestUtil.deleteHttpEventCollectorToken(httpEventCollectorName); } - } diff --git a/src/test/java/testEventbodySerializer.java b/src/test/java/testEventbodySerializer.java new file mode 100644 index 00000000..2422718c --- /dev/null +++ b/src/test/java/testEventbodySerializer.java @@ -0,0 +1,41 @@ +/* + * Copyright 2013-2014 Splunk, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import com.splunk.logging.EventBodySerializer; +import com.splunk.logging.HttpEventCollectorErrorHandler; +import com.splunk.logging.HttpEventCollectorEventInfo; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.logging.Logger; + +// Implement the interface of EventBodySerializer for testing +public class testEventBodySerializer implements EventBodySerializer { + + @Override + public String serializeEventBody( + final HttpEventCollectorEventInfo eventInfo, + final Object formattedMessage + ) { + return "user-prefix:" + formattedMessage.toString(); + } +} + diff --git a/src/test/resources/logging_template.properties b/src/test/resources/logging_template.properties index b96aaaac..be9285a6 100644 --- a/src/test/resources/logging_template.properties +++ b/src/test/resources/logging_template.properties @@ -20,6 +20,8 @@ com.splunk.logging.HttpEventCollectorLoggingHandler.disableCertificateValidation com.splunk.logging.HttpEventCollectorLoggingHandler.send_mode=%user_send_mode% com.splunk.logging.HttpEventCollectorLoggingHandler.retries_on_error = %user_retries_on_error% +com.splunk.logging.HttpEventCollectorLoggingHandler.eventBodySerializer = %user_eventBodySerializer% + # You would usually use XMLFormatter or SimpleFormatter for this property, but # SimpleFormatter doesn't accept a format string under Java 6, and so we cannot # control its output. Thus we use a trivial formatter as part of the test suite From 9804c93bfc0e6b675c68a0cae2fb4d01bc401bea Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 13:20:14 -0700 Subject: [PATCH 6/8] fix compile error --- src/test/java/HttpEventCollector_JavaLoggingTest.java | 2 +- src/test/java/testEventbodySerializer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/HttpEventCollector_JavaLoggingTest.java b/src/test/java/HttpEventCollector_JavaLoggingTest.java index ff667921..1ba3b967 100644 --- a/src/test/java/HttpEventCollector_JavaLoggingTest.java +++ b/src/test/java/HttpEventCollector_JavaLoggingTest.java @@ -462,7 +462,7 @@ public void canSendEventUsingJavaLoggingWithUserEventBodySerializer() throws Exc HashMap userInputs = new HashMap(); userInputs.put("user_httpEventCollector_token", token); userInputs.put("user_logger_name", loggerName); - userInputs.put("user_eventBodySerializer", "testEventBodySerializer"); + userInputs.put("user_eventBodySerializer", "TestEventBodySerializer"); TestUtil.resetJavaLoggingConfiguration("logging_template.properties", "logging.properties", userInputs); diff --git a/src/test/java/testEventbodySerializer.java b/src/test/java/testEventbodySerializer.java index 2422718c..73bb7625 100644 --- a/src/test/java/testEventbodySerializer.java +++ b/src/test/java/testEventbodySerializer.java @@ -28,7 +28,7 @@ import java.util.logging.Logger; // Implement the interface of EventBodySerializer for testing -public class testEventBodySerializer implements EventBodySerializer { +public class TestEventBodySerializer implements EventBodySerializer { @Override public String serializeEventBody( From 0f5305a962020340ba700c64b1e88a952bbb86e8 Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Wed, 24 Oct 2018 13:43:32 -0700 Subject: [PATCH 7/8] fix compile error --- ...{testEventbodySerializer.java => TestEventBodySerializer.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/test/java/{testEventbodySerializer.java => TestEventBodySerializer.java} (100%) diff --git a/src/test/java/testEventbodySerializer.java b/src/test/java/TestEventBodySerializer.java similarity index 100% rename from src/test/java/testEventbodySerializer.java rename to src/test/java/TestEventBodySerializer.java From 4d6191e6b9269739d21547c72edb1e2a3489566b Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Tue, 30 Oct 2018 14:22:19 -0700 Subject: [PATCH 8/8] Updated version to 1.6.2 --- CHANGELOG.md | 6 ++++++ README.md | 2 +- pom.xml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29ffa7e3..8a398f8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Splunk Logging for Java Changelog +## Version 1.6.2 + +* Add support to allow users to define their own event body serializer for HTTP event adapter: Simply create a class implementing `com.splunk.logging.EventBodySerializer`, +and add the full class name as a property (`eventBodySerializer`) to the adapter. +Default will be a JSON event body containing message, severity, and other properties. [#86](https://github.com/splunk/splunk-library-javalogging/pull/86). + ## Version 1.6.1 * TcpAppender performance improvement, prevents 100% CPU usage [#85](https://github.com/splunk/splunk-library-javalogging/pull/85). diff --git a/README.md b/README.md index 5edc79fb..f6dd01ca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Splunk Logging for Java -#### Version 1.6.1 +#### Version 1.6.2 This project provides utilities to easily log data using Splunk's recommended best practices to any supported logger, using any of the three major Java diff --git a/pom.xml b/pom.xml index f487beb7..3250c6a3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.splunk.logging splunk-library-javalogging - 1.6.1 + 1.6.2 jar Splunk Logging for Java