Merge pull request #425 from splunk/release/7_2_1

Release eventgen 7.2.1. Merge to master branch.

docs/CHANGELOG.md

@@ -1,3 +1,7 @@
+**7.2.1**:
+
+- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.2.1)
+
 **7.2.0**:
 
 - Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.2.0)

docs/Gemfile.lock

@@ -33,7 +33,7 @@ GEM
     github-pages (201)
       activesupport (= 4.2.11.1)
       github-pages-health-check (= 1.16.1)
-      jekyll (= 3.8.5)
+      jekyll (= 3.9.0)
       jekyll-avatar (= 0.6.0)
       jekyll-coffeescript (= 1.1.1)
       jekyll-commonmark-ghpages (= 0.1.6)
@@ -67,7 +67,7 @@ GEM
       jekyll-theme-time-machine (= 0.1.1)
       jekyll-titles-from-headings (= 0.5.1)
       jemoji (= 0.10.2)
-      kramdown (= 1.17.0)
+      kramdown (>= 2.3.0)
       liquid (= 4.0.0)
       listen (= 3.1.5)
       mercenary (~> 0.3)
@@ -87,14 +87,14 @@ GEM
     http_parser.rb (0.6.0)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    jekyll (3.8.5)
+    jekyll (3.9.0)
       addressable (~> 2.4)
       colorator (~> 1.0)
       em-websocket (~> 0.5)
       i18n (~> 0.7)
       jekyll-sass-converter (~> 1.0)
       jekyll-watch (~> 2.0)
-      kramdown (~> 1.14)
+      kramdown (>= 1.17, < 3)
       liquid (~> 4.0)
       mercenary (~> 0.3.3)
       pathutil (~> 0.9)
@@ -192,30 +192,34 @@ GEM
       gemoji (~> 3.0)
       html-pipeline (~> 2.2)
       jekyll (~> 3.0)
-    kramdown (1.17.0)
+    kramdown (2.3.0)
+      rexml
     liquid (4.0.0)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
       ruby_dep (~> 1.2)
     mercenary (0.3.6)
-    mini_portile2 (2.4.0)
+    mini_portile2 (2.5.0)
     minima (2.5.0)
       jekyll (~> 3.5)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
     minitest (5.12.2)
     multipart-post (2.1.1)
-    nokogiri (1.10.8)
-      mini_portile2 (~> 2.4.0)
+    nokogiri (1.11.0)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     octokit (4.14.0)
       sawyer (~> 0.8.0, >= 0.5.3)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (3.1.1)
+    racc (1.5.2)
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
+    rexml (3.2.4)
     rouge (3.11.0)
     ruby-enum (0.7.2)
       i18n

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api"
 
 [tool.poetry]
 name = "splunk_eventgen"
-version = "7.2.0"
+version = "7.2.1"
 description = "Splunk Event Generator to produce real-time, representative data"
 authors = [ "Brian Bingham <bbingham@splunk.com>", "Tony Lee <tonyl@splunk.com>", "Jack Meixensperger <jackm@splunk.com>",]
 license = "Apache-2.0"

splunk_eventgen/__main__.py

@@ -274,6 +274,8 @@ def build_splunk_app(dest, source=os.getcwd(), remove=True):
     target_file = os.path.join(dest, "sa_eventgen_{}.spl".format(EVENTGEN_VERSION))
     splunk_app = os.path.join(FILE_LOCATION, "splunk_app")
     splunk_app_samples = os.path.join(splunk_app, "samples")
+    if os.path.exists(splunk_app_samples):
+        shutil.rmtree(splunk_app_samples)
     shutil.copytree(os.path.join(FILE_LOCATION, "samples"), splunk_app_samples)
     try:
         shutil.copytree(splunk_app, directory)
@@ -291,7 +293,7 @@ def build_splunk_app(dest, source=os.getcwd(), remove=True):
     shutil.copyfile(eventgen_conf, directory_default_dir)
 
     # install 3rd lib dependencies
-    install_target = os.path.join(directory, "lib")
+    install_target = os.path.join(directory, "bin")
     install_cmd = (
         "pip install --requirement splunk_eventgen/lib/requirements.txt --upgrade --no-compile "
         + "--no-binary :all: --target "

splunk_eventgen/eventgen_core.py

@@ -266,9 +266,7 @@ def _create_generator_pool(self, workercount=20):
                     worker = Thread(
                         target=self._generator_do_work,
                         args=(self.workerQueue, self.loggingQueue),
-                        kwargs={
-                            "output_counter": self.output_counters[i],
-                        },
+                        kwargs={"output_counter": self.output_counters[i]},
                     )
                     worker.setDaemon(True)
                     worker.start()
@@ -277,9 +275,7 @@ def _create_generator_pool(self, workercount=20):
                     worker = Thread(
                         target=self._generator_do_work,
                         args=(self.workerQueue, self.loggingQueue),
-                        kwargs={
-                            "output_counter": None,
-                        },
+                        kwargs={"output_counter": None},
                     )
                     worker.setDaemon(True)
                     worker.start()

splunk_eventgen/lib/eventgentimer.py

@@ -182,7 +182,6 @@ def real_run(self):
                                 raweventsize=raw_event_size,
                             )
                             self.perdayrater.rate()
-                            self.perdayrater.queue_it(count)
                         self.rater.queue_it(count)
                     self.countdown = self.interval
                     self.executions += 1

splunk_eventgen/lib/generatorplugin.py

@@ -33,16 +33,8 @@ def __str__(self):
     def __repr__(self):
         return self.__str__()
 
-    def build_events(
-        self, eventsDict, startTime, earliest, latest, ignore_tokens=False
-    ):
+    def send_events(self, send_objects, startTime):
         """Ready events for output by replacing tokens and updating the output queue"""
-        # Replace tokens first so that perDayVolume evaluates the correct event length
-        send_objects = self.replace_tokens(
-            eventsDict, earliest, latest, ignore_tokens=ignore_tokens
-        )
-        # after replace_tokens() is called, we don't need eventsDict
-        del eventsDict
         try:
             self._out.bulksend(send_objects)
             self._sample.timestamp = None

splunk_eventgen/lib/logging_config/__init__.py

@@ -21,10 +21,7 @@
     },
     "filters": {},
     "handlers": {
-        "console": {
-            "class": "logging.StreamHandler",
-            "formatter": "default",
-        },
+        "console": {"class": "logging.StreamHandler", "formatter": "default"},
         "eventgen_main": {
             "class": "logging.handlers.RotatingFileHandler",
             "formatter": "default",

splunk_eventgen/lib/plugins/generator/default.py

@@ -73,7 +73,8 @@ def gen(self, count, earliest, latest, samplename=None):
                     % (self._sample.name, self._sample.app, len(eventsDict))
                 )
 
-        GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest)
+        send_objects = self.replace_tokens(eventsDict, earliest, latest)
+        self.send_events(send_objects, startTime)
 
 
 def load():

splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py

@@ -33,6 +33,21 @@ def gen(self, count, earliest, latest, samplename=None):
         )
         startTime = datetime.datetime.now()
 
+        # Pre-generate each event in the sample once, so we can replace tokens and calculate any volume differences
+        preReplacementSize = sum(
+            [len(event["_raw"]) for event in self._sample.sampleDict]
+        )
+        allEvents = self.replace_tokens(self._sample.sampleDict, earliest, latest)
+        postReplacementSize = sum([len(event["_raw"]) for event in allEvents])
+        replacementVolumeRatio = preReplacementSize / postReplacementSize
+        size = size * replacementVolumeRatio
+        logger.debug(
+            "Token replacement size factor: {}, new interval target size: {}".format(
+                replacementVolumeRatio, size
+            )
+        )
+        del allEvents
+
         # Create a counter for the current byte size of the read in samples
         currentSize = 0
 
@@ -94,8 +109,10 @@ def gen(self, count, earliest, latest, samplename=None):
                 % (self._sample.name, self._sample.app, len(eventsDict))
             )
 
-        # build the events and replace tokens
-        self.build_events(eventsDict, startTime, earliest, latest)
+        # replace tokens and send events
+        send_objects = self.replace_tokens(eventsDict, earliest, latest)
+        del eventsDict
+        self.send_events(send_objects, startTime)
 
 
 def load():

splunk_eventgen/splunk_app/README.md

@@ -0,0 +1,13 @@
+### Introduction
+
+SA-Eventgen (Splunk App Eventgen) allows users to generate and index custom event data on their Splunk instance.
+This app is currently built and maintained through the [Eventgen repository](https://github.com/splunk/eventgen).
+
+### Documentation
+
+The general Eventgen documentation is hosted on [Github](http://splunk.github.io/eventgen).
+For installation and configuration instructions specific to the Splunk App, see the
+[app install](http://splunk.github.io/eventgen/SETUP.html#splunk-app-installation) page of the documentation.
+
+Finally, see to the [Eventgen Reference Guide](http://splunk.github.io/eventgen/REFERENCE.html#eventgenconfspec)
+for information on Eventgen configuration options.

splunk_eventgen/splunk_app/README/eventgen.conf.spec

@@ -17,7 +17,6 @@
 ## IMPORTANT! Do not specify any settings under a default stanza
 ## The layering system will not behave appropriately
 ## Use [global] instead
-[default]
 
 [global]
 disabled = false

splunk_eventgen/splunk_app/README/inputs.conf.spec

@@ -1,2 +1,3 @@
 [modinput_eventgen://<name>]
 verbosity = <value>
+python.version = python3

splunk_eventgen/splunk_app/default/app.conf

@@ -8,13 +8,13 @@
 ## Splunk app configuration file
 
 [install]
-is_configured = true
+is_configured = false
 state = enabled
 build = 1
 
 [launcher]
 author = Splunk Inc.
-version = 7.2.0
+version = 7.2.1
 description = SA-Eventgen app for dynamic data generation
 
 [package]
@@ -23,3 +23,7 @@ id = SA-Eventgen
 [ui]
 is_visible = true
 label = SA-Eventgen
+
+[triggers]
+reload.eventgen = simple
+reload.inputs.eventgen = access_endpoints /data/inputs/eventgen

tests/large/conf/eventgen_perdayvolume.conf

@@ -0,0 +1,6 @@
+[sample]
+sampleDir = ../sample
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_perdayvolume.result
+perDayVolume = 1

tests/large/conf/eventgen_perdayvolume_large_token.conf

@@ -0,0 +1,10 @@
+[sample]
+sampleDir = ../sample
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_perdayvolume.result
+perDayVolume = 1
+
+token.0.token = @@integer
+token.0.replacementType = static
+token.0.replacement = "supercalifragilisticexpialidocious"

tests/large/conf/eventgen_perdayvolume_small_token.conf

@@ -0,0 +1,10 @@
+[sample]
+sampleDir = ../sample
+sampletype = raw
+outputMode = file
+fileName = tests/large/results/eventgen_perdayvolume.result
+perDayVolume = 1
+
+token.0.token = @@integer
+token.0.replacementType = random
+token.0.replacement = integer[0:10]

tests/large/test_perdayvolume.py

@@ -0,0 +1,30 @@
+from pytest import mark
+
+
+def calculate_perdayvolume(events, runtime, interval=60):
+    # Calculate expected data volume output (GB) if run for 24 hours
+    # Get the integer # of intervals, data is only generated on completion of an interval
+    num_intervals = runtime // interval
+    event_volume = sum([len(event) for event in events])
+    total_volume = event_volume / 1024 / 1024 / 1024 * 60 * 24
+    perdayvolume = total_volume / num_intervals
+    return perdayvolume
+
+
+@mark.parametrize(
+    ("conf_filename", "execution_timeout", "perdayvolume"),
+    [
+        ("eventgen_perdayvolume.conf", 300, 1),
+        ("eventgen_perdayvolume_small_token.conf", 300, 1),
+        ("eventgen_perdayvolume_large_token.conf", 300, 1),
+    ],
+)
+def test_perdayvolume(
+    eventgen_test_helper, conf_filename, execution_timeout, perdayvolume
+):
+    # Test accuracy of small volume target with no token replacements
+    # TODO: using outputMode=file for now, test helper unable to collect all generated events w/ outputMode=stdout
+    events = eventgen_test_helper(conf_filename, execution_timeout).get_events()
+    assert (
+        0.98 < (calculate_perdayvolume(events, execution_timeout) / perdayvolume) < 1.02
+    )