Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/161 subsecond decimals #172

Merged
merged 9 commits into from Mar 26, 2024
Merged

Feature/161 subsecond decimals #172

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
734392e
Fixes #161 add subsecond decimal configuration
EEParker Mar 16, 2024
72d2528
Fixes #167 Add a flag for RenderedMessage
EEParker Mar 16, 2024
ac79391
Update SubSecondPrecision to use an enum.
EEParker Mar 16, 2024
d1bce24
Add unit tests and update documentation
EEParker Mar 16, 2024
b7cd9f7
Invert renderMessage check
LordZeroHour Mar 19, 2024
b335663
Update sample docker-compose project
EEParker Mar 25, 2024
b0e77ad
Add comment for configuring splunk
EEParker Mar 26, 2024
867db7c
Fix formatting for nanoseconds
EEParker Mar 26, 2024
e9c5d09
Enable all tests in sample app
EEParker Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/Serilog.Sinks.Splunk/Sinks/Splunk/CompactSplunkJsonFormatter.cs
View file
Open in desktop
Expand Up @@ -31,6 +31,7 @@ public class CompactSplunkJsonFormatter : ITextFormatter
private static readonly JsonValueFormatter ValueFormatter = new JsonValueFormatter(typeTagName: "$type");
private readonly string _suffix;
private readonly bool _renderTemplate;
private readonly int _subSecondDecimals;

/// <summary>
/// Construct a <see cref="CompactSplunkJsonFormatter"/>.
Expand All @@ -40,9 +41,13 @@ public class CompactSplunkJsonFormatter : ITextFormatter
/// <param name="host">The host of the event</param>
/// <param name="index">The Splunk index to log to</param>
/// <param name="renderTemplate">If true, the template used will be rendered and written to the output as a property named MessageTemplate</param>
public CompactSplunkJsonFormatter(bool renderTemplate = false, string source = null, string sourceType = null, string host = null, string index = null)
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>

public CompactSplunkJsonFormatter(bool renderTemplate = false, string source = null, string sourceType = null, string host = null, string index = null, int subSecondDecimals = 3)
{
_renderTemplate = renderTemplate;
_subSecondDecimals = subSecondDecimals;

var suffixWriter = new StringWriter();
suffixWriter.Write("}"); // Terminates "event"

Expand Down Expand Up @@ -80,7 +85,7 @@ public void Format(LogEvent logEvent, TextWriter output)
if (output == null) throw new ArgumentNullException(nameof(output));

output.Write("{\"time\":\"");
output.Write(logEvent.Timestamp.ToEpoch().ToString(CultureInfo.InvariantCulture));
output.Write(logEvent.Timestamp.ToEpoch(_subSecondDecimals).ToString(CultureInfo.InvariantCulture));
output.Write("\",\"event\":{\"@l\":\"");
output.Write(logEvent.Level);
output.Write('"');
Expand Down
4 changes: 2 additions & 2 deletions src/Serilog.Sinks.Splunk/Sinks/Splunk/Epoch.cs
View file
Open in desktop
Expand Up @@ -20,15 +20,15 @@ internal static class EpochExtensions
{
private static DateTimeOffset Epoch = new DateTimeOffset(1970,1,1,0,0,0,TimeSpan.Zero);

public static double ToEpoch(this DateTimeOffset value)
public static double ToEpoch(this DateTimeOffset value, int subSecondDecimals = 3)
{
// From Splunk HTTP Collector Protocol
// The default time format is epoch time format, in the format <sec>.<ms>.
// For example, 1433188255.500 indicates 1433188255 seconds and 500 milliseconds after epoch,
// or Monday, June 1, 2015, at 7:50:55 PM GMT.
// See: http://dev.splunk.com/view/SP-CAAAE6P

return Math.Round((value - Epoch).TotalSeconds, 3, MidpointRounding.AwayFromZero);
return Math.Round((value - Epoch).TotalSeconds, subSecondDecimals, MidpointRounding.AwayFromZero);
}
}
}
35 changes: 24 additions & 11 deletions src/Serilog.Sinks.Splunk/Sinks/Splunk/EventCollectorSink.cs
View file
Open in desktop
Expand Up @@ -12,17 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.

using Serilog.Debugging;
using Serilog.Events;
using Serilog.Formatting;
using Serilog.Sinks.PeriodicBatching;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Serilog.Debugging;
using Serilog.Events;
using Serilog.Formatting;
using Serilog.Sinks.PeriodicBatching;

namespace Serilog.Sinks.Splunk
{
Expand Down Expand Up @@ -56,17 +56,23 @@ public class EventCollectorSink : IBatchedLogEventSink
/// <param name="eventCollectorToken">The token to use when authenticating with the event collector</param>
/// <param name="formatProvider">The format provider used when rendering the message</param>
/// <param name="renderTemplate">Whether to render the message template</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
public EventCollectorSink(
string splunkHost,
string eventCollectorToken,
IFormatProvider formatProvider = null,
bool renderTemplate = true)
bool renderTemplate = true,
bool renderMessage = true,
int subSecondDecimals = 3)
: this(
splunkHost,
eventCollectorToken,
null, null, null, null, null,
formatProvider,
renderTemplate)
renderTemplate,
renderMessage,
subSecondDecimals: subSecondDecimals)
{
}

Expand All @@ -83,6 +89,8 @@ public class EventCollectorSink : IBatchedLogEventSink
/// <param name="sourceType">The source type of the event</param>
/// <param name="host">The host of the event</param>
/// <param name="messageHandler">The handler used to send HTTP requests</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
public EventCollectorSink(
string splunkHost,
string eventCollectorToken,
Expand All @@ -93,13 +101,14 @@ public class EventCollectorSink : IBatchedLogEventSink
string index,
IFormatProvider formatProvider = null,
bool renderTemplate = true,
HttpMessageHandler messageHandler = null)
bool renderMessage = true,
HttpMessageHandler messageHandler = null,
int subSecondDecimals = 3)
: this(
splunkHost,
eventCollectorToken,
uriPath,

new SplunkJsonFormatter(renderTemplate, formatProvider, source, sourceType, host, index),
new SplunkJsonFormatter(renderTemplate, renderMessage, formatProvider, source, sourceType, host, index, subSecondDecimals: subSecondDecimals),
messageHandler)
{
}
Expand All @@ -118,6 +127,8 @@ public class EventCollectorSink : IBatchedLogEventSink
/// <param name="sourceType">The source type of the event</param>
/// <param name="host">The host of the event</param>
/// <param name="messageHandler">The handler used to send HTTP requests</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
public EventCollectorSink(
string splunkHost,
string eventCollectorToken,
Expand All @@ -129,13 +140,15 @@ public class EventCollectorSink : IBatchedLogEventSink
CustomFields fields,
IFormatProvider formatProvider = null,
bool renderTemplate = true,
HttpMessageHandler messageHandler = null)
bool renderMessage = true,
HttpMessageHandler messageHandler = null,
int subSecondDecimals = 3)
// TODO here is the jsonformatter creation. We must make way to test output of jsonformatter.
: this(
splunkHost,
eventCollectorToken,
uriPath,
new SplunkJsonFormatter(renderTemplate, formatProvider, source, sourceType, host, index, fields),
new SplunkJsonFormatter(renderTemplate, renderMessage, formatProvider, source, sourceType, host, index, fields, subSecondDecimals: subSecondDecimals),
messageHandler)
{
}
Expand Down
31 changes: 24 additions & 7 deletions src/Serilog.Sinks.Splunk/Sinks/Splunk/SplunkJsonFormatter.cs
View file
Open in desktop
Expand Up @@ -32,6 +32,8 @@ public class SplunkJsonFormatter : ITextFormatter

private readonly bool _renderTemplate;
private readonly IFormatProvider _formatProvider;
private readonly int _subSecondDecimals;
private readonly bool _renderMessage;
private readonly string _suffix;

/// <inheritdoc />
Expand All @@ -40,10 +42,12 @@ public class SplunkJsonFormatter : ITextFormatter
/// </summary>
/// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
/// <param name="renderTemplate">If true, the template used will be rendered and written to the output as a property named MessageTemplate</param>
/// <param name="renderMessage">Removes the "RenderedMessage" parameter from output JSON message.</param>
public SplunkJsonFormatter(
bool renderTemplate,
bool renderMessage,
IFormatProvider formatProvider)
: this(renderTemplate, formatProvider, null, null, null, null)
: this(renderTemplate, renderMessage, formatProvider, null, null, null, null)
{
}

Expand All @@ -56,14 +60,18 @@ public class SplunkJsonFormatter : ITextFormatter
/// <param name="source">The source of the event</param>
/// <param name="sourceType">The source type of the event</param>
/// <param name="host">The host of the event</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
/// <param name="renderMessage">Removes the "RenderedMessage" parameter from output JSON message.</param>
public SplunkJsonFormatter(
bool renderTemplate,
bool renderMessage,
IFormatProvider formatProvider,
string source,
string sourceType,
string host,
string index)
: this(renderTemplate, formatProvider, source, sourceType, host, index, null)
string index,
int subSecondDecimals = 3)
: this(renderTemplate, renderMessage, formatProvider, source, sourceType, host, index, null, subSecondDecimals: subSecondDecimals)
{
}

Expand All @@ -77,17 +85,23 @@ public class SplunkJsonFormatter : ITextFormatter
/// <param name="sourceType">The source type of the event</param>
/// <param name="host">The host of the event</param>
/// <param name="customFields">Object that describes extra splunk fields that should be indexed with event see: http://dev.splunk.com/view/event-collector/SP-CAAAFB6 </param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
public SplunkJsonFormatter(
bool renderTemplate,
bool renderMessage,
IFormatProvider formatProvider,
string source,
string sourceType,
string host,
string index,
CustomFields customFields)
CustomFields customFields,
int subSecondDecimals = 3)
{
_renderTemplate = renderTemplate;
_formatProvider = formatProvider;
_subSecondDecimals = subSecondDecimals;
_renderMessage = renderMessage;

using (var suffixWriter = new StringWriter())
{
Expand Down Expand Up @@ -157,7 +171,7 @@ public void Format(LogEvent logEvent, TextWriter output)
if (output == null) throw new ArgumentNullException(nameof(output));

output.Write("{\"time\":\"");
output.Write(logEvent.Timestamp.ToEpoch().ToString(CultureInfo.InvariantCulture));
output.Write(logEvent.Timestamp.ToEpoch(_subSecondDecimals).ToString(CultureInfo.InvariantCulture));
output.Write("\",\"event\":{\"Level\":\"");
output.Write(logEvent.Level);
output.Write('"');
Expand All @@ -168,8 +182,11 @@ public void Format(LogEvent logEvent, TextWriter output)
JsonValueFormatter.WriteQuotedJsonString(logEvent.MessageTemplate.Text, output);
}

output.Write(",\"RenderedMessage\":");
JsonValueFormatter.WriteQuotedJsonString(logEvent.RenderMessage(_formatProvider), output);
if (!_renderMessage)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this logic be the other way round? ie if (_renderMessage) to match the _renderTemplate logic above?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you are right. I removed the !

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the fix

{
output.Write(",\"RenderedMessage\":");
JsonValueFormatter.WriteQuotedJsonString(logEvent.RenderMessage(_formatProvider), output);
}

if (logEvent.Exception != null)
{
Expand Down
24 changes: 18 additions & 6 deletions src/Serilog.Sinks.Splunk/SplunkLoggingConfigurationExtensions.cs
View file
Open in desktop
Expand Up @@ -17,7 +17,6 @@
using Serilog.Core;
using Serilog.Events;
using Serilog.Formatting;
using Serilog.Formatting.Json;
using Serilog.Sinks.PeriodicBatching;
using Serilog.Sinks.Splunk;
using System;
Expand All @@ -44,11 +43,13 @@ public static class SplunkLoggingConfigurationExtensions
/// <param name="restrictedToMinimumLevel">The minimum log event level required in order to write an event to the sink.</param>
/// <param name="formatProvider">Supplies culture-specific formatting information, or null.</param>
/// <param name="renderTemplate">If true, the message template will be rendered</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
/// <param name="batchIntervalInSeconds">The interval in seconds that the queue should be instpected for batching</param>
/// <param name="batchSizeLimit">The size of the batch</param>
/// <param name="queueLimit">Maximum number of events in the queue</param>
/// <param name="messageHandler">The handler used to send HTTP requests</param>
/// <param name="levelSwitch">A switch allowing the pass-through minimum level to be changed at runtime.</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
/// <returns></returns>
public static LoggerConfiguration EventCollector(
this LoggerSinkConfiguration configuration,
Expand All @@ -62,11 +63,13 @@ public static class SplunkLoggingConfigurationExtensions
LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
IFormatProvider formatProvider = null,
bool renderTemplate = true,
bool renderMessage = true,
int batchIntervalInSeconds = 2,
int batchSizeLimit = 100,
int? queueLimit = null,
HttpMessageHandler messageHandler = null,
LoggingLevelSwitch levelSwitch = null)
LoggingLevelSwitch levelSwitch = null,
int subSecondDecimals = 3)
{
if (configuration == null) throw new ArgumentNullException(nameof(configuration));

Expand All @@ -88,7 +91,9 @@ public static class SplunkLoggingConfigurationExtensions
index,
formatProvider,
renderTemplate,
messageHandler);
renderMessage,
subSecondDecimals: subSecondDecimals);

var batchingSink = new PeriodicBatchingSink(eventCollectorSink, batchingOptions);

return configuration.Sink(batchingSink, restrictedToMinimumLevel, levelSwitch);
Expand All @@ -103,7 +108,6 @@ public static class SplunkLoggingConfigurationExtensions
/// <param name="jsonFormatter">The text formatter used to render log events into a JSON format for consumption by Splunk</param>
/// <param name="uriPath">Change the default endpoint of the Event Collector e.g. services/collector/event</param>
/// <param name="restrictedToMinimumLevel">The minimum log event level required in order to write an event to the sink.</param>

/// <param name="batchIntervalInSeconds">The interval in seconds that the queue should be instpected for batching</param>
/// <param name="batchSizeLimit">The size of the batch</param>
/// <param name="queueLimit">Maximum number of events in the queue</param>
Expand Down Expand Up @@ -168,6 +172,8 @@ public static class SplunkLoggingConfigurationExtensions
/// <param name="messageHandler">The handler used to send HTTP requests</param>
/// <param name="levelSwitch">A switch allowing the pass-through minimum level to be changed at runtime.</param>
/// <param name="fields">Customfields that will be indexed in splunk with this event</param>
/// <param name="renderMessage">Include "RenderedMessage" parameter from output JSON message.</param>
/// <param name="subSecondDecimals">Timestamp sub-second precision</param>
/// <returns></returns>
public static LoggerConfiguration EventCollector(
this LoggerSinkConfiguration configuration,
Expand All @@ -182,11 +188,13 @@ public static class SplunkLoggingConfigurationExtensions
LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
IFormatProvider formatProvider = null,
bool renderTemplate = true,
bool renderMessage = true,
int batchIntervalInSeconds = 2,
int batchSizeLimit = 100,
int? queueLimit = null,
HttpMessageHandler messageHandler = null,
LoggingLevelSwitch levelSwitch = null)
LoggingLevelSwitch levelSwitch = null,
int subSecondDecimals = 3)
{
if (configuration == null) throw new ArgumentNullException(nameof(configuration));

Expand All @@ -209,7 +217,11 @@ public static class SplunkLoggingConfigurationExtensions
fields,
formatProvider,
renderTemplate,
messageHandler);
renderMessage,
messageHandler,
subSecondDecimals: subSecondDecimals
);


var batchingSink = new PeriodicBatchingSink(eventCollectorSink, batchingOptions);

Expand Down