This is the second part of Datawave #716 issue.

pom.xml

@@ -8,7 +8,7 @@
         <relativePath />
     </parent>
     <artifactId>metadata-utils</artifactId>
-    <version>1.8-SNAPSHOT</version>
+    <version>1.8-zeiberg-716-1.0</version>
     <url>https://code.nsa.gov/datawave-metadata-utils</url>
     <licenses>
         <license>
@@ -19,7 +19,7 @@
     <scm>
         <connection>scm:git:https://github.com/NationalSecurityAgency/datawave-metadata-utils.git</connection>
         <developerConnection>scm:git:git@github.com:NationalSecurityAgency/datawave-metadata-utils.git</developerConnection>
-        <tag>HEAD</tag>
+        <tag>1.8-zeiberg-716-1.0</tag>
         <url>https://github.com/NationalSecurityAgency/datawave-metadata-utils</url>
     </scm>
     <properties>

src/main/java/datawave/query/util/DateFrequencyValue.java

@@ -0,0 +1,268 @@
+package datawave.query.util;
+
+import com.google.common.base.Preconditions;
+import org.apache.accumulo.core.data.Value;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.time.Year;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * This class handles the serialization and deserialization of the Accumulo value in a record of the Datawave Metadata table that has a column family of "f" and
+ * a column qualifier that is prefixed with the string "compressed-" like "compressed-csv" for example. This is a class used to help compress the date and
+ * frequency values that are aggregated together to by the FrequencyTransformIterator and manipulated in the FrequencyFamilyCounter. The byte array is in
+ * regular expression format ((YEAR)(4BYTE-FREQUENCY){366}))* . Explained verbally a four byte representation of Year followed by by up to 366 (Leap year) 4
+ * byte holders for frequency values. The month and day of the frequency value is coded by the position in the array. There aren't any delimiters between years
+ * and frequencies which adds to the compression. Each Accumulo row for this "aggregated" frequency "map" would be 10 x ( 4 + 4 + (366 * 4) ) bytes long for a
+ * maximum length for a 10 year capture of 14720 bytes.
+ */
+
+public class DateFrequencyValue {
+
+    private static final Logger log = LoggerFactory.getLogger(DateFrequencyValue.class);
+
+    private static int DAYS_IN_LEAP_YEAR = YearMonthDay.DAYS_IN_LEAP_YEAR;
+    private static int NUM_YEAR_BYTES = 4;
+    private static int NUM_BYTES_PER_FREQ_VALUE = 4;
+    private static int NUM_FREQUENCY_BYTES = DAYS_IN_LEAP_YEAR * NUM_BYTES_PER_FREQ_VALUE;
+
+    public DateFrequencyValue() {}
+
+    /**
+     * @param dateToFrequencyValueMap
+     *            the keys should be dates in yyyyMMdd format
+     * @return size the size of the ByteArrayOutputStream
+     */
+    private int calculateOutputArraySize(TreeMap<YearMonthDay,Frequency> dateToFrequencyValueMap) {
+        int firstYear = dateToFrequencyValueMap.firstKey().getYear();
+        int lastYear = dateToFrequencyValueMap.lastKey().getYear();
+        return (lastYear - firstYear + 1) * (NUM_YEAR_BYTES + NUM_FREQUENCY_BYTES);
+    }
+
+    /**
+     *
+     * Serializes a treemap of dates with associated frequencies to an Accumulo value
+     *
+     * @param dateToFrequencyValueMap
+     *            the keys should be dates in yyyyMMdd format
+     * @return Value the value to store in accumulo
+     */
+    public Value serialize(TreeMap<YearMonthDay,Frequency> dateToFrequencyValueMap) {
+
+        Value serializedMap;
+        int year, presentYear = 0;
+        ByteArrayOutputStream baos = new ByteArrayOutputStream(calculateOutputArraySize(dateToFrequencyValueMap));
+        int ordinal, nextOrdinal = 1;
+
+        for (Map.Entry<YearMonthDay,Frequency> dateFrequencyEntry : dateToFrequencyValueMap.entrySet()) {
+            year = dateFrequencyEntry.getKey().getYear();
+            ordinal = dateFrequencyEntry.getKey().getJulian();
+
+            if (year != presentYear) {
+
+                if (nextOrdinal > 1 && (nextOrdinal <= DAYS_IN_LEAP_YEAR)) {
+                    // Add zero frequencies for the remaining days of the year that were not in the map
+                    // for the last year processed
+                    do {
+                        Base256Compression.writeToOutputStream(0, baos);
+                        nextOrdinal++;
+                    } while (nextOrdinal <= DAYS_IN_LEAP_YEAR);
+                }
+
+                Base256Compression.writeToOutputStream(year, baos);
+                nextOrdinal = 1;
+                presentYear = year;
+            }
+
+            if (ordinal == nextOrdinal) {
+                Base256Compression.writeToOutputStream(dateFrequencyEntry.getValue().value, baos);
+                nextOrdinal++;
+            } else {
+                do {
+                    Base256Compression.writeToOutputStream(0, baos);
+                    nextOrdinal++;
+                } while (nextOrdinal < ordinal);
+
+                Base256Compression.writeToOutputStream(dateFrequencyEntry.getValue().value, baos);
+                nextOrdinal++;
+
+            }
+
+            if (log.isTraceEnabled())
+                log.trace(dateFrequencyEntry.getKey().toString());
+
+        }
+
+        do {
+            Base256Compression.writeToOutputStream(0, baos);
+            nextOrdinal++;
+        } while (nextOrdinal <= DAYS_IN_LEAP_YEAR);
+
+        serializedMap = new Value(baos.toByteArray());
+
+        return serializedMap;
+    }
+
+    /**
+     * Deserializes the Accumulo Value object which contains a byte array into a TreeMap of dates to the associated ingest frequencies.
+     *
+     * @param oldValue
+     * @return
+     */
+    public TreeMap<YearMonthDay,Frequency> deserialize(Value oldValue) {
+        return deserialize(new TreeMap(), oldValue, null, false, null, false);
+    }
+
+    /**
+     * Deserializes the Accumulo Value object which contains a byte array into a TreeMap of dates to the associated ingest frequencies.
+     *
+     * @param oldValue
+     * @param startDate
+     *            yyyyMMdd start date to deserialize, null means no bound
+     * @param startInclusive
+     * @param endDate
+     *            yyyyMMdd end date to deserialize, null means no bound
+     * @param endInclusive
+     * @return
+     */
+    public TreeMap<YearMonthDay,Frequency> deserialize(Value oldValue, String startDate, boolean startInclusive, String endDate, boolean endInclusive) {
+        return deserialize(new TreeMap(), oldValue, startDate, startInclusive, endDate, endInclusive);
+    }
+
+    /**
+     * Deserializes and aggregates the Accumulo Value object which contains a byte array into a TreeMap of dates to the associated ingest frequencies.
+     *
+     * @param dateFrequencyMap
+     * @param oldValue
+     * @param startDate
+     *            yyyyMMdd start date to deserialize, null means no bound
+     * @param startInclusive
+     * @param endDate
+     *            yyyyMMdd end date to deserialize, null means no bound
+     * @param endInclusive
+     * @return
+     */
+    public TreeMap<YearMonthDay,Frequency> deserialize(TreeMap<YearMonthDay,Frequency> dateFrequencyMap, Value oldValue, String startDate,
+                    boolean startInclusive, String endDate, boolean endInclusive) {
+        Preconditions.checkNotNull(oldValue);
+
+        byte[] expandedData = oldValue.get();
+
+        if (expandedData.length == 0) {
+            return dateFrequencyMap;
+        }
+
+        YearMonthDay.Bounds bounds = new YearMonthDay.Bounds(startDate, startInclusive, endDate, endInclusive);
+
+        try {
+            for (int i = 0; i < expandedData.length; i += (NUM_YEAR_BYTES + NUM_FREQUENCY_BYTES)) {
+                int decodedYear = Base256Compression.bytesToInteger(expandedData[i], expandedData[i + 1], expandedData[i + 2], expandedData[i + 3]);
+                String decodedYearStr = String.valueOf(decodedYear);
+                log.debug("Deserialize decoded the year " + decodedYearStr);
+
+                if (bounds.intersectsYear(decodedYearStr)) {
+                    // determine the range within this year
+                    int startOrdinalInYear = bounds.getStartOrdinal(decodedYearStr);
+                    int endOrdinalInYear = bounds.getEndOrdinal(decodedYearStr);
+
+                    // Decode the frequencies for each day of the year within the range.
+                    for (int j = NUM_YEAR_BYTES + ((startOrdinalInYear - 1) * NUM_BYTES_PER_FREQ_VALUE); j < endOrdinalInYear * NUM_BYTES_PER_FREQ_VALUE
+                                    + NUM_YEAR_BYTES; j += NUM_BYTES_PER_FREQ_VALUE) {
+                        int k = i + j;
+                        YearMonthDay date = new YearMonthDay(decodedYear, ((j - NUM_YEAR_BYTES) / NUM_BYTES_PER_FREQ_VALUE) + 1);
+                        int decodedFrequencyOnDay = Base256Compression.bytesToInteger(expandedData[k], expandedData[k + 1], expandedData[k + 2],
+                                        expandedData[k + 3]);
+                        if (decodedFrequencyOnDay != 0) {
+                            if (dateFrequencyMap.containsKey(date)) {
+                                dateFrequencyMap.get(date).addFrequency(decodedFrequencyOnDay);
+                            } else {
+                                dateFrequencyMap.put(date, new Frequency(decodedFrequencyOnDay));
+                            }
+                            log.debug("put key value pair in SimpleDateFrequency map: " + date + ", " + dateFrequencyMap.get(date));
+                        }
+                    }
+                }
+            }
+        } catch (IndexOutOfBoundsException indexOutOfBoundsException) {
+            log.error("Error decoding the compressed array of date values. Expanded array length: " + expandedData.length, indexOutOfBoundsException);
+            throw new IllegalStateException("Error decoding the compressed array of date values. Expanded array length: " + expandedData.length,
+                            indexOutOfBoundsException);
+        }
+
+        return dateFrequencyMap;
+    }
+
+    /**
+     * Sums the frequencies of the Accumulo Value object which contains a byte array into a TreeMap of dates to the associated ingest frequencies.
+     *
+     * @param oldValue
+     * @param startDate
+     *            yyyyMMdd start date to deserialize, null means no bound
+     * @param startInclusive
+     * @param endDate
+     *            yyyyMMdd end date to deserialize, null means no bound
+     * @param endInclusive
+     * @return
+     */
+    public long count(Value oldValue, String startDate, boolean startInclusive, String endDate, boolean endInclusive) {
+        Preconditions.checkNotNull(oldValue);
+
+        byte[] expandedData = oldValue.get();
+
+        if (expandedData.length == 0) {
+            return 0;
+        }
+
+        YearMonthDay.Bounds bounds = new YearMonthDay.Bounds(startDate, startInclusive, endDate, endInclusive);
+
+        try {
+            long count = 0;
+            for (int i = 0; i < expandedData.length; i += (NUM_YEAR_BYTES + NUM_FREQUENCY_BYTES)) {
+                int decodedYear = Base256Compression.bytesToInteger(expandedData[i], expandedData[i + 1], expandedData[i + 2], expandedData[i + 3]);
+                String decodedYearStr = String.valueOf(decodedYear);
+                log.debug("Deserialize decoded the year " + decodedYearStr);
+
+                if (bounds.intersectsYear(decodedYearStr)) {
+                    // determine the range within this year
+                    int startOrdinalInYear = bounds.getStartOrdinal(decodedYearStr);
+                    int endOrdinalInYear = bounds.getEndOrdinal(decodedYearStr);
+
+                    // Decode the frequencies for each day of the year within the range.
+                    for (int j = NUM_YEAR_BYTES + ((startOrdinalInYear - 1) * NUM_BYTES_PER_FREQ_VALUE); j < endOrdinalInYear * NUM_BYTES_PER_FREQ_VALUE
+                                    + NUM_YEAR_BYTES; j += NUM_BYTES_PER_FREQ_VALUE) {
+                        int k = i + j;
+                        YearMonthDay date = new YearMonthDay(decodedYear, ((j - NUM_YEAR_BYTES) / NUM_BYTES_PER_FREQ_VALUE) + 1);
+                        count += Base256Compression.bytesToInteger(expandedData[k], expandedData[k + 1], expandedData[k + 2], expandedData[k + 3]);
+                    }
+                }
+            }
+            return count;
+        } catch (IndexOutOfBoundsException indexOutOfBoundsException) {
+            log.error("Error decoding the compressed array of date values. Expanded array length: " + expandedData.length, indexOutOfBoundsException);
+            throw new IllegalStateException("Error decoding the compressed array of date values. Expanded array length: " + expandedData.length,
+                            indexOutOfBoundsException);
+        }
+    }
+
+    /**
+     * This is a helper class that will compress the yyyyMMdd and the frequency date concatenated to it without a delimiter
+     */
+    public static class Base256Compression {
+
+        public static void writeToOutputStream(long num, ByteArrayOutputStream baos) {
+            baos.write((byte) (num >>> 24));
+            baos.write((byte) (num >>> 16));
+            baos.write((byte) (num >>> 8));
+            baos.write((byte) (num));
+        }
+
+        public static int bytesToInteger(byte high, byte nextHigh, byte lowbyte, byte lowest) {
+            return ((int) high & 0xff) << 24 | ((int) nextHigh & 0xff) << 16 | ((int) lowbyte & 0xff) << 8 | ((int) lowest & 0xff);
+        }
+
+    }
+
+}

src/main/java/datawave/query/util/Frequency.java

@@ -0,0 +1,36 @@
+package datawave.query.util;
+
+public class Frequency {
+    int value;
+
+    public Frequency(int value) {
+        this.value = value;
+    }
+
+    public int getValue() {
+        return value;
+    }
+
+    void addFrequency(int value) {
+        this.value += value;
+    }
+
+    @Override
+    public String toString() {
+        return "Frequency{" + "value=" + value + '}';
+    }
+
+    @Override
+    public int hashCode() {
+        return value;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof Frequency) {
+            if (((Frequency) obj).value == value)
+                return true;
+        }
+        return false;
+    }
+}