Switch Kff to use exclusively SBCs

src/main/java/emissary/core/IBaseDataObjectHelper.java

@@ -11,7 +11,6 @@
 
 import java.io.IOException;
 import java.lang.reflect.Field;
-import java.security.NoSuchAlgorithmException;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -173,11 +172,7 @@ public static void addParentInformationToChild(final IBaseDataObject parentIBase
         KffDataObjectHandler.parentToChild(childIBaseDataObject);
 
         // Hash the new child data, overwrites parent hashes if any
-        try {
-            kffDataObjectHandler.hash(childIBaseDataObject, true);
-        } catch (NoSuchAlgorithmException | IOException e) {
-            // Do not add the hash parameters
-        }
+        kffDataObjectHandler.hash(childIBaseDataObject);
     }
 
     /**

src/main/java/emissary/core/channels/SeekableByteChannelHelper.java

@@ -19,6 +19,9 @@
 public final class SeekableByteChannelHelper {
     private static final Logger logger = LoggerFactory.getLogger(SeekableByteChannelHelper.class);
 
+    /** Channel factory backed by an empty byte array. Used for situations when a BDO should have its payload discarded. */
+    public static final SeekableByteChannelFactory EMPTY_CHANNEL_FACTORY = memory(new byte[0]);
+
     private SeekableByteChannelHelper() {}
 
     /**

src/main/java/emissary/kff/KffDataObjectHandler.java

@@ -2,6 +2,7 @@
 
 import emissary.core.IBaseDataObject;
 import emissary.core.channels.SeekableByteChannelFactory;
+import emissary.core.channels.SeekableByteChannelHelper;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,59 +71,22 @@ public KffDataObjectHandler(boolean truncateKnownData, boolean setFormOnKnownDat
      * Compute the configure hashes and return as a map Also include entries indicating the know file or duplicate file
      * status if so configured
      * 
-     * @param data the bytes to hash
-     * @param name th name of the data (for reporting)
+     * @param sbcf the data to hash
+     * @param name the name of the data (for reporting)
      * @return parameter entries suitable for a BaseDataObject
+     * @throws IOException if the data can't be read
+     * @throws NoSuchAlgorithmException if the checksum can't be computed
      */
-    public Map<String, String> hashData(byte[] data, String name) {
-        return hashData(data, name, "");
-    }
-
-    /**
-     * Compute the configure hashes and return as a map Also include entries indicating the know file or duplicate file
-     * status if so configured
-     * 
-     * @param data the bytes to hash
-     * @param name th name of the data (for reporting)
-     * @param prefix prepended to hash name entries
-     * @return parameter entries suitable for a BaseDataObject
-     */
-    public Map<String, String> hashData(@Nullable byte[] data, String name, @Nullable String prefix) {
-        Map<String, String> results = new HashMap<String, String>();
-
-        if (prefix == null) {
-            prefix = "";
-        }
-
-        if (data != null && data.length > 0) {
-            try {
-                KffResult kffCheck = kff.check(name, data);
-
-                // Store all computed results in data object params
-                for (String alg : kffCheck.getResultNames()) {
-                    results.put(prefix + KFF_PARAM_BASE + alg, kffCheck.getResultString(alg));
-                }
-
-                // Set params if we have a hit
-                if (kffCheck.isKnown()) {
-                    results.put(prefix + KFF_PARAM_KNOWN_FILTER_NAME, kffCheck.getFilterName());
-                }
-                if (kffCheck.isDupe()) {
-                    results.put(prefix + KFF_PARAM_DUPE_FILTER_NAME, kffCheck.getFilterName());
-                }
-            } catch (Exception kffex) {
-                logger.warn("Unable to compute kff on " + name, kffex);
-            }
-        }
-        return results;
+    public Map<String, String> hashData(final SeekableByteChannelFactory sbcf, final String name) throws NoSuchAlgorithmException, IOException {
+        return hashData(sbcf, name, "");
     }
 
     /**
      * Compute the configure hashes and return as a map Also include entries indicating the know file or duplicate file
      * status if so configured
      * 
      * @param sbcf the data to hash
-     * @param name th name of the data (for reporting)
+     * @param name the name of the data (for reporting)
      * @param prefix prepended to hash name entries
      * @return parameter entries suitable for a BaseDataObject
      * @throws IOException if the data can't be read
@@ -166,22 +130,6 @@ public Map<String, String> hashData(final SeekableByteChannelFactory sbcf, final
      * @param d the data object
      */
     public void hash(@Nullable final IBaseDataObject d) {
-        try {
-            hash(d, false);
-        } catch (NoSuchAlgorithmException | IOException e) {
-            // Do nothing
-        }
-    }
-
-    /**
-     * Compute the hash of a data object's data
-     * 
-     * @param d the data object
-     * @param useSbc use the {@link SeekableByteChannel} interface
-     * @throws IOException if the data can't be read
-     * @throws NoSuchAlgorithmException if the checksum can't be computed
-     */
-    public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws NoSuchAlgorithmException, IOException {
         if (d != null) {
             removeHash(d);
         }
@@ -190,13 +138,15 @@ public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws
             return;
         }
 
-        // Compute and add the hashes
-        if (useSbc && d.getChannelSize() > 0) {
-            d.putParameters(hashData(d.getChannelFactory(), d.shortName(), ""));
-        } else if (!useSbc && d.dataLength() > 0) {
-            d.putParameters(hashData(d.data(), d.shortName()));
-        } else {
-            return;
+        try {
+            // Compute and add the hashes
+            if (d.getChannelSize() > 0) {
+                d.putParameters(hashData(d.getChannelFactory(), d.shortName()));
+            } else {
+                return;
+            }
+        } catch (NoSuchAlgorithmException | IOException e) {
+            logger.error("Couldn't hash data {}", d.shortName());
         }
 
         // Set params if we have a hit
@@ -208,7 +158,7 @@ public void hash(@Nullable final IBaseDataObject d, final boolean useSbc) throws
                 d.replaceCurrentForm(KFF_DUPE_CURRENT_FORM);
             }
             if (truncateKnownData) {
-                d.setData(null);
+                d.setChannelFactory(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY);
             }
         }
     }

src/main/java/emissary/place/KffHashPlace.java

@@ -6,7 +6,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.security.NoSuchAlgorithmException;
 
 /**
  * Hashing place to hash payload unless hashes are set or skip flag is set. This place is intended to execute in the
@@ -22,8 +21,6 @@ public class KffHashPlace extends ServiceProviderPlace {
 
     public static final String SKIP_KFF_HASH = "SKIP_KFF_HASH";
 
-    private boolean useSbc = false;
-
     public KffHashPlace(String thePlaceLocation) throws IOException {
         super(thePlaceLocation);
     }
@@ -51,7 +48,6 @@ public KffHashPlace(InputStream configStream, String placeLocation) throws IOExc
     @Override
     protected void setupPlace(String theDir, String placeLocation) throws IOException {
         super.setupPlace(theDir, placeLocation);
-        useSbc = configG.findBooleanEntry("USE_SBC", useSbc);
         initKff();
     }
 
@@ -62,11 +58,7 @@ public void process(IBaseDataObject payload) throws ResourceException {
             return;
         }
 
-        try {
-            kff.hash(payload, useSbc);
-        } catch (final NoSuchAlgorithmException | IOException e) {
-            logger.error("KffHashPlace failed to hash data for {} - this shouldn't happen", payload.shortName(), e);
-        }
+        kff.hash(payload);
     }
 
 }

src/main/resources/emissary/place/KffHashPlace.cfg

@@ -6,6 +6,3 @@ SERVICE_COST = 10
 SERVICE_QUALITY = 100
 
 SERVICE_PROXY = "UNKNOWN"
-
-# Use SeekableByteChannel-related methods (true) vs byte array based (false)
-USE_SBC = false

src/test/java/emissary/core/IBaseDataObjectHelperTest.java

@@ -1,6 +1,7 @@
 package emissary.core;
 
 import emissary.core.channels.InMemoryChannelFactory;
+import emissary.core.channels.SeekableByteChannelFactory;
 import emissary.kff.KffDataObjectHandler;
 import emissary.parser.SessionParser;
 
@@ -317,7 +318,8 @@ void testAddParentInformationToChild() throws Exception {
         final IBaseDataObject childIbdo1 = new BaseDataObject();
 
         childIbdo1.setChannelFactory(InMemoryChannelFactory.create("0123456789".getBytes(StandardCharsets.US_ASCII)));
-        Mockito.doThrow(NoSuchAlgorithmException.class).when(mockKffDataObjectHandler1).hash(Mockito.any(BaseDataObject.class), Mockito.anyBoolean());
+        Mockito.doThrow(NoSuchAlgorithmException.class).when(mockKffDataObjectHandler1).hashData(Mockito.any(SeekableByteChannelFactory.class),
+                Mockito.anyString());
         IBaseDataObjectHelper.addParentInformationToChild(parentIbdo, childIbdo1,
                 true, alwaysCopyMetadataKeys, placeKey, mockKffDataObjectHandler1);
         assertFalse(KffDataObjectHandler.hashPresent(childIbdo1));

src/test/java/emissary/kff/KffDataObjectHandlerTest.java

@@ -2,6 +2,9 @@
 
 import emissary.core.DataObjectFactory;
 import emissary.core.IBaseDataObject;
+import emissary.core.channels.AbstractSeekableByteChannel;
+import emissary.core.channels.SeekableByteChannelFactory;
+import emissary.core.channels.SeekableByteChannelHelper;
 import emissary.test.core.junit5.UnitTest;
 import emissary.util.io.ResourceReader;
 
@@ -11,17 +14,24 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 class KffDataObjectHandlerTest extends UnitTest {
-    static final byte[] DATA = "This is a test".getBytes();
+    static final SeekableByteChannelFactory DATA = SeekableByteChannelHelper.memory("This is a test".getBytes());
 
     // echo -n "This is a test" | openssl sha1
     static final String DATA_SHA1 = "a54d88e06612d820bc3be72877c74f257b561b19";
@@ -72,19 +82,19 @@ public void tearDown() throws Exception {
     }
 
     @Test
-    void testMapWithEmptyPrefix() {
+    void testMapWithEmptyPrefix() throws NoSuchAlgorithmException, IOException {
         Map<String, String> m = kff.hashData(DATA, "junk");
         assertNotNull(m.get(KffDataObjectHandler.KFF_PARAM_MD5), "Empty prefix returns normal values");
     }
 
     @Test
-    void testMapWithNullPrefix() {
+    void testMapWithNullPrefix() throws NoSuchAlgorithmException, IOException {
         Map<String, String> m = kff.hashData(DATA, "junk", null);
         assertNotNull(m.get(KffDataObjectHandler.KFF_PARAM_MD5), "Null prefix returns normal values");
     }
 
     @Test
-    void testMapWithPrefix() {
+    void testMapWithPrefix() throws NoSuchAlgorithmException, IOException {
         Map<String, String> m = kff.hashData(DATA, "name", "foo");
         assertNotNull(m.get("foo" + KffDataObjectHandler.KFF_PARAM_MD5), "Prefix prepends on normal key names but we got " + m.keySet());
     }
@@ -131,4 +141,107 @@ void testSetAndGetHash() {
         payload.deleteParameter(KffDataObjectHandler.KFF_PARAM_SHA512);
         assertEquals(DATA_SHA384, KffDataObjectHandler.getBestAvailableHash(payload));
     }
+
+    @Test
+    void testWithChannelFactory() {
+        kff = new KffDataObjectHandler(true, true, true);
+        payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
+        payload.setChannelFactory(DATA);
+        kff.hash(payload);
+        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
+        assertTrue(KffDataObjectHandler.hashPresent(payload));
+        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
+        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
+        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
+        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
+        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
+        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());
+        assertArrayEquals(new byte[0], payload.data());
+    }
+
+    @Test
+    void testWithEmptyChannelFactory() {
+        kff = new KffDataObjectHandler(true, true, true);
+        payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
+        payload.setChannelFactory(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY);
+        kff.hash(payload);
+        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
+        assertFalse(KffDataObjectHandler.hashPresent(payload));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
+        assertEquals("test", payload.getFileType());
+        assertArrayEquals(new byte[0], payload.data());
+        assertEquals(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY, payload.getChannelFactory());
+    }
+
+    @Test
+    void testNullPayload() {
+        assertDoesNotThrow(() -> kff.hash(null));
+    }
+
+    @Test
+    void testRemovingHash() {
+        final SeekableByteChannelFactory exceptionSbcf = new SeekableByteChannelFactory() {
+
+            @Override
+            public SeekableByteChannel create() {
+                return new AbstractSeekableByteChannel() {
+
+                    @Override
+                    protected void closeImpl() throws IOException {
+                        // Do nothing
+                    }
+
+                    @Override
+                    protected int readImpl(ByteBuffer byteBuffer) throws IOException {
+                        throw new IOException("Test exception");
+                    }
+
+                    @Override
+                    protected long sizeImpl() throws IOException {
+                        throw new IOException("Test exception");
+                    }
+
+                };
+            }
+
+        };
+
+        payload.setChannelFactory(exceptionSbcf);
+        kff.hash(payload);
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
+        assertFalse(KffDataObjectHandler.hashPresent(payload));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
+        assertNull(payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
+        assertNotEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());
+
+        payload.setParameter(KffDataObjectHandler.KFF_PARAM_KNOWN_FILTER_NAME, "test.filter");
+        payload.setChannelFactory(DATA);
+        kff.hash(payload);
+        assertEquals("test.filter", payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "FILTERED_BY"));
+        assertTrue(KffDataObjectHandler.hashPresent(payload));
+        assertEquals(DATA_MD5, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_MD5));
+        assertEquals(DATA_CRC32, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_BASE + "CRC32"));
+        assertEquals(DATA_SSDEEP, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SSDEEP));
+        assertEquals(DATA_SHA1, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA1));
+        assertEquals(DATA_SHA256, payload.getStringParameter(KffDataObjectHandler.KFF_PARAM_SHA256));
+        assertEquals(KffDataObjectHandler.KFF_DUPE_CURRENT_FORM, payload.getFileType());
+
+    }
+
+    @Test
+    void testNullHashData() throws NoSuchAlgorithmException, IOException {
+        assertEquals(new HashMap<>(), kff.hashData(null, null));
+    }
+
+    @Test
+    void testEmptySbcf() throws NoSuchAlgorithmException, IOException {
+        assertEquals(new HashMap<>(), kff.hashData(SeekableByteChannelHelper.EMPTY_CHANNEL_FACTORY, null));
+    }
 }