From 96eaa0eaa09c5901dfed0ffbc27c28a463db26e6 Mon Sep 17 00:00:00 2001
From: timrobertson100 <timrobertson100@gmail.com>
Date: Wed, 11 Dec 2019 15:27:12 +0100
Subject: [PATCH] #159 Skeleton code for Index Herbariorum synchronization

---
 pom.xml                                       |   1 +
 registry-collections-sync/README.md           |   3 +
 registry-collections-sync/pom.xml             |  95 ++++++++++++
 .../registry/collections/sync/GRSciColl.java  | 101 ++++++++++++
 .../collections/sync/IndexHerbariorum.java    |  88 +++++++++++
 .../sync/SyncIndexHerbariorum.java            | 145 ++++++++++++++++++
 6 files changed, 433 insertions(+)
 create mode 100644 registry-collections-sync/README.md
 create mode 100644 registry-collections-sync/pom.xml
 create mode 100644 registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java
 create mode 100644 registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java
 create mode 100644 registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/SyncIndexHerbariorum.java

diff --git a/pom.xml b/pom.xml
index bdfed0caaf..e680c623ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,6 +60,7 @@
     <module>registry-surety</module>
     <module>registry-ws</module>
     <module>registry-ws-client</module>
+    <module>registry-collections-sync</module>
   </modules>
 
   <!--
diff --git a/registry-collections-sync/README.md b/registry-collections-sync/README.md
new file mode 100644
index 0000000000..6c10432dca
--- /dev/null
+++ b/registry-collections-sync/README.md
@@ -0,0 +1,3 @@
+# Registry Collections Synchronisation
+
+Provides synchronisation utilities for key repositories such as Index Hebrarirum.
diff --git a/registry-collections-sync/pom.xml b/registry-collections-sync/pom.xml
new file mode 100644
index 0000000000..f39449e48e
--- /dev/null
+++ b/registry-collections-sync/pom.xml
@@ -0,0 +1,95 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.gbif.registry</groupId>
+    <artifactId>registry-motherpom</artifactId>
+    <version>2.140-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>registry-collections-sync</artifactId>
+  <packaging>jar</packaging>
+
+  <name>Registry Collections Sync</name>
+
+  <distributionManagement>
+    <site>
+      <id>gh-pages</id>
+      <url>http://gbif.github.io/registry/${project.artifactId}/</url>
+    </site>
+  </distributionManagement>
+
+  <properties>
+    <slf4j.version>1.7.29</slf4j.version>
+    <jackson.version>2.9.4</jackson.version>
+    <retrofit.version>2.5.0</retrofit.version>
+    <okhttp.version>3.11.0</okhttp.version>
+    <okio.version>2.0.0</okio.version>
+    <lombok.version>1.18.10</lombok.version>
+  </properties>
+
+  <repositories>
+    <repository>
+      <id>gbif-all</id>
+      <url>http://repository.gbif.org/content/groups/gbif</url>
+    </repository>
+    <repository>
+      <id>gbif-thirdparty</id>
+      <url>http://repository.gbif.org/content/repositories/thirdparty/</url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>gbif-api</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.retrofit2</groupId>
+      <artifactId>retrofit</artifactId>
+      <version>${retrofit.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.retrofit2</groupId>
+      <artifactId>converter-jackson</artifactId>
+      <version>${retrofit.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <version>${okhttp.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio</artifactId>
+      <version>${okio.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>${lombok.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>
diff --git a/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java
new file mode 100644
index 0000000000..16343030f3
--- /dev/null
+++ b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java
@@ -0,0 +1,101 @@
+package org.gbif.registry.collections.sync;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.gbif.api.model.collections.Collection;
+import org.gbif.api.model.collections.Institution;
+import org.gbif.api.model.common.paging.PagingResponse;
+import org.gbif.api.vocabulary.Country;
+import retrofit2.Call;
+import retrofit2.Retrofit;
+import retrofit2.converter.jackson.JacksonConverterFactory;
+import retrofit2.http.GET;
+import retrofit2.http.Query;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A lightweight GRSciColl client.
+ */
+public class GRSciColl {
+  private static API api = newClient();
+
+  /**
+   * Returns all institutions in GrSciCol.
+   */
+  static List<Institution> institutions() throws IOException {
+    List<Institution> result = new ArrayList<>();
+    boolean endRecords = false;
+    int offset = 0;
+    while (!endRecords) {
+      PagingResponse<Institution> response = api.listInstitutions(1000, offset).execute().body();
+      endRecords = response.isEndOfRecords();
+      offset += response.getLimit();
+      result.addAll(response.getResults());
+    }
+    return result;
+  }
+
+  /**
+   * Returns all institutions in GrSciCol.
+   */
+  static List<Collection> collections() throws IOException {
+    List<Collection> result = new ArrayList<>();
+    boolean endRecords = false;
+    int offset = 0;
+    while (!endRecords) {
+      PagingResponse<Collection> response = api.listCollections(1000, offset).execute().body();
+      endRecords = response.isEndOfRecords();
+      offset += response.getLimit();
+      result.addAll(response.getResults());
+    }
+    return result;
+  }
+
+  private static API newClient() {
+    ObjectMapper mapper = new ObjectMapper();
+    SimpleModule module = new SimpleModule();
+    module.addDeserializer(Country.class, new IsoDeserializer());
+    mapper.registerModule(module);
+
+    Retrofit retrofit = new Retrofit.Builder()
+      .baseUrl("http://api.gbif.org/v1/grscicoll/") // TODO parameterize
+      .addConverterFactory(JacksonConverterFactory.create(mapper))
+      .build();
+    return retrofit.create(API.class);
+  }
+
+  private interface API {
+    @GET("institution")
+    Call<PagingResponse<Institution>> listInstitutions(@Query("limit") int limit, @Query("offset") int offset);
+
+    @GET("collection")
+    Call<PagingResponse<Collection>> listCollections(@Query("limit") int limit, @Query("offset") int offset);
+  }
+
+  /**
+   * Adapter necessary for retrofit due to versioning.
+   */
+  private static class IsoDeserializer extends JsonDeserializer<Country> {
+    @Override
+    public Country deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      try {
+        if (jp != null && jp.getTextLength() > 0) {
+          return Country.fromIsoCode(jp.getText());
+        } else {
+          return Country.UNKNOWN; // none provided
+        }
+      } catch (Exception e) {
+        throw new IOException("Unable to deserialize country from provided value (not an ISO 2 character?): "
+          + jp.getText());
+      }
+    }
+  }
+
+
+}
diff --git a/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java
new file mode 100644
index 0000000000..40d9da8acc
--- /dev/null
+++ b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java
@@ -0,0 +1,88 @@
+package org.gbif.registry.collections.sync;
+
+import lombok.Data;
+import retrofit2.Call;
+import retrofit2.Retrofit;
+import retrofit2.converter.jackson.JacksonConverterFactory;
+import retrofit2.http.GET;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Lightweight IndexHerbariorum client.
+ */
+class IndexHerbariorum {
+
+  static List<IndexHerbariorum.Institution> institutions() throws IOException {
+    return newClient().listInstitutions().execute().body().getData();
+  }
+
+
+  private static API newClient() {
+    Retrofit retrofit = new Retrofit.Builder()
+      //.baseUrl("http://sweetgum.nybg.org/science/api/v1/") // TODO: Parameterize
+      .baseUrl("http://labs.gbif.org/") // go easy on them...
+      .addConverterFactory(JacksonConverterFactory.create())
+      .build();
+    return retrofit.create(API.class);
+  }
+
+  private interface API {
+    @GET("institutions")
+    Call<InstitutionWrapper> listInstitutions();
+  }
+
+  @Data
+  private static class InstitutionWrapper {
+    private Meta meta;
+    private List<Institution> data;
+
+    @Data
+    private static class Meta {
+      private int hits;
+      private int code;
+    }
+  }
+
+
+  @Data
+  static class Institution {
+    private String irn;
+    private String organization;
+    private String code;
+    private String division;
+    private String department;
+    private long specimenTotal;
+    private Address address;
+    private Contact contact;
+    private Location location;
+    private String dateModified;
+
+    @Data
+    static class Address {
+      private String physicalStreet;
+      private String physicalCity;
+      private String physicalState;
+      private String physicalZipCode;
+      private String physicalCountry;
+      private String postalStreet;
+      private String postalCity;
+      private String postalState;
+      private String postalZipCode;
+      private String postalCountry;
+    }
+
+    @Data
+    static class Contact {
+      private String phone;
+      private String email;
+      private String webUrl;
+    }
+
+    @Data
+    static class Location {
+      private Double lat;
+      private Double lon;
+    }
+  }
+}
diff --git a/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/SyncIndexHerbariorum.java b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/SyncIndexHerbariorum.java
new file mode 100644
index 0000000000..6879026592
--- /dev/null
+++ b/registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/SyncIndexHerbariorum.java
@@ -0,0 +1,145 @@
+package org.gbif.registry.collections.sync;
+
+import lombok.ToString;
+import org.gbif.api.model.collections.Collection;
+import org.gbif.api.model.collections.Institution;
+import org.gbif.api.model.registry.Identifiable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * A synchronization utility that will ensure GRSciColl is up to date with IndexHerbariorum.
+ * This operates as follows:
+ * <ul>
+ *   <li>Retrieve all Herbaria from IndexHerbariorum</li>
+ *   <li>For each entity locate the equivalent Institution or Collection in GRSciColl using the IH IRN</li>
+ *   <li>If the entity exists and they differ, update GrSciColl</li>
+ *   <li>If the entity does not exist, insert it as an institution and with an identifier holding the IH IRN</li>
+ * </ul>
+ * <p>A future version of this may allow editing of IH entities in GRSciColl. Under that scenario when entities differ
+ * more complex logic is required, likely requiring notification to GRSciColl and IH staff to resolve the differences.
+ * <p>TODO: Add synchronisation of staff
+ */
+public class SyncIndexHerbariorum {
+  private static Logger LOG = LoggerFactory.getLogger(SyncIndexHerbariorum.class);
+  
+  public static void main(String[] args) throws IOException {
+
+    LOG.info("Loading IH");
+    List<IndexHerbariorum.Institution> herbaria = IndexHerbariorum.institutions();
+    LOG.info("Loading Institutions");
+    List<Institution> institutions = GRSciColl.institutions();
+    LOG.info("Loading Collections");
+    List<Collection> collections = GRSciColl.collections();
+
+    Counter counter = new Counter();
+    for (IndexHerbariorum.Institution herbarium : herbaria) {
+      counter.total++;
+
+      // locate potential matches in GrSciColl
+      Set<Institution> matchedInstitutions = filterById(institutions, encodeIRN(herbarium.getIrn()));
+      Set<Collection> matchedCollections = filterById(collections, encodeIRN(herbarium.getIrn()));
+
+      if (matchedInstitutions.size()==1 && matchedCollections.isEmpty()) {
+        Institution existing = matchedInstitutions.iterator().next();
+        Institution institution = buildInstitution(herbarium);
+        if (!institution.lenientEquals(existing)) {
+          counter.institutionUpdated++;
+          LOG.info("Updating institution: {}", institution.getName());
+        } else {
+          counter.institutionNoChange++;
+          LOG.info("Skipping institution [no change]: {}", institution.getName());
+        }
+
+      } else if (matchedCollections.size()==1 && matchedInstitutions.isEmpty()) {
+        Collection existing = matchedCollections.iterator().next();
+        Collection collection = buildCollection(herbarium, existing.getInstitutionKey());
+        if (!collection.lenientEquals(existing)) {
+          counter.collectionUpdated++;
+          LOG.info("Updating collection: {}", collection.getName());
+        } else {
+          counter.collectionNoChange++;
+          LOG.info("Skipping collection [no change]: {}", collection.getName());
+        }
+
+
+      } else if (matchedInstitutions.isEmpty() && matchedCollections.isEmpty()) {
+        counter.institutionCreated++;
+        Institution institution = buildInstitution(herbarium);
+        LOG.info("Creating new institution: {}", institution.getName());
+        // remember to add the identifier!
+
+      } else {
+        counter.conflict++;
+        // Conflict that needs resolved manually
+        LOG.info("Conflict. {} institutions and {} collections are candidate matches in registry: ",
+          herbarium.getOrganization());
+
+      }
+
+      LOG.debug("{}: {}", counter.total, herbarium.getOrganization());
+    }
+    LOG.info("Synchronisation finished. {}", counter);
+  }
+
+  @ToString
+  private static class Counter {
+    int total;
+    int institutionUpdated;
+    int institutionCreated;
+    int institutionNoChange;
+    int collectionUpdated;
+    int collectionNoChange;
+    int conflict;
+  }
+
+  /**
+   * Creates an institution from the IH object.
+   */
+  private static Institution buildInstitution(IndexHerbariorum.Institution ih) {
+    Institution i = new Institution();
+    i.setName(ih.getOrganization());
+    // TODO
+    return i;
+  }
+
+  /**
+   * Creates a collection from the IH object.
+   */
+  private static Collection buildCollection(IndexHerbariorum.Institution ih, UUID institutionKey) {
+    Collection c = new Collection();
+    c.setInstitutionKey(institutionKey);
+    c.setName(ih.getOrganization());
+    // TODO
+    return c;
+  }
+
+  /**
+   * Encodes the IH IRN into the format stored on the GRSciColl identifier.
+   * E.g. 123 -> gbif:ih:irn:123
+   */
+  private static String encodeIRN(String irn) {
+    return "gbif:ih:irn:" + irn;
+  }
+
+  /**
+   * Filters the source to only those having the given ID.
+   */
+  private static <T extends Identifiable> Set<T> filterById(List<T> source, String id) {
+    return source.stream()
+      .filter(
+        o ->
+          o.getIdentifiers().stream()
+            .filter(i -> Objects.equals(id, i.getIdentifier()))
+            .count()
+            > 0)
+      .collect(Collectors.toSet());
+  }
+
+}