#159 Skeleton code for Index Herbariorum synchronization

pom.xml

@@ -60,6 +60,7 @@
     <module>registry-surety</module>
     <module>registry-ws</module>
     <module>registry-ws-client</module>
+    <module>registry-collections-sync</module>
   </modules>
 
   <!--

registry-collections-sync/README.md

@@ -0,0 +1,3 @@
+# Registry Collections Synchronisation
+
+Provides synchronisation utilities for key repositories such as Index Hebrarirum.

registry-collections-sync/pom.xml

@@ -0,0 +1,95 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.gbif.registry</groupId>
+    <artifactId>registry-motherpom</artifactId>
+    <version>2.140-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>registry-collections-sync</artifactId>
+  <packaging>jar</packaging>
+
+  <name>Registry Collections Sync</name>
+
+  <distributionManagement>
+    <site>
+      <id>gh-pages</id>
+      <url>http://gbif.github.io/registry/${project.artifactId}/</url>
+    </site>
+  </distributionManagement>
+
+  <properties>
+    <slf4j.version>1.7.29</slf4j.version>
+    <jackson.version>2.9.4</jackson.version>
+    <retrofit.version>2.5.0</retrofit.version>
+    <okhttp.version>3.11.0</okhttp.version>
+    <okio.version>2.0.0</okio.version>
+    <lombok.version>1.18.10</lombok.version>
+  </properties>
+
+  <repositories>
+    <repository>
+      <id>gbif-all</id>
+      <url>http://repository.gbif.org/content/groups/gbif</url>
+    </repository>
+    <repository>
+      <id>gbif-thirdparty</id>
+      <url>http://repository.gbif.org/content/repositories/thirdparty/</url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.gbif</groupId>
+      <artifactId>gbif-api</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.retrofit2</groupId>
+      <artifactId>retrofit</artifactId>
+      <version>${retrofit.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.retrofit2</groupId>
+      <artifactId>converter-jackson</artifactId>
+      <version>${retrofit.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <version>${okhttp.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio</artifactId>
+      <version>${okio.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>${lombok.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>

registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java

@@ -0,0 +1,101 @@
+package org.gbif.registry.collections.sync;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.gbif.api.model.collections.Collection;
+import org.gbif.api.model.collections.Institution;
+import org.gbif.api.model.common.paging.PagingResponse;
+import org.gbif.api.vocabulary.Country;
+import retrofit2.Call;
+import retrofit2.Retrofit;
+import retrofit2.converter.jackson.JacksonConverterFactory;
+import retrofit2.http.GET;
+import retrofit2.http.Query;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A lightweight GRSciColl client.
+ */
+public class GRSciColl {
+  private static API api = newClient();
+
+  /**
+   * Returns all institutions in GrSciCol.
+   */
+  static List<Institution> institutions() throws IOException {
+    List<Institution> result = new ArrayList<>();
+    boolean endRecords = false;
+    int offset = 0;
+    while (!endRecords) {
+      PagingResponse<Institution> response = api.listInstitutions(1000, offset).execute().body();
+      endRecords = response.isEndOfRecords();
+      offset += response.getLimit();
+      result.addAll(response.getResults());
+    }
+    return result;
+  }
+
+  /**
+   * Returns all institutions in GrSciCol.
+   */
+  static List<Collection> collections() throws IOException {
+    List<Collection> result = new ArrayList<>();
+    boolean endRecords = false;
+    int offset = 0;
+    while (!endRecords) {
+      PagingResponse<Collection> response = api.listCollections(1000, offset).execute().body();
+      endRecords = response.isEndOfRecords();
+      offset += response.getLimit();
+      result.addAll(response.getResults());
+    }
+    return result;
+  }
+
+  private static API newClient() {
+    ObjectMapper mapper = new ObjectMapper();
+    SimpleModule module = new SimpleModule();
+    module.addDeserializer(Country.class, new IsoDeserializer());
+    mapper.registerModule(module);
+
+    Retrofit retrofit = new Retrofit.Builder()
+      .baseUrl("http://api.gbif.org/v1/grscicoll/") // TODO parameterize
+      .addConverterFactory(JacksonConverterFactory.create(mapper))
+      .build();
+    return retrofit.create(API.class);
+  }
+
+  private interface API {
+    @GET("institution")
+    Call<PagingResponse<Institution>> listInstitutions(@Query("limit") int limit, @Query("offset") int offset);
+
+    @GET("collection")
+    Call<PagingResponse<Collection>> listCollections(@Query("limit") int limit, @Query("offset") int offset);
+  }
+
+  /**
+   * Adapter necessary for retrofit due to versioning.
+   */
+  private static class IsoDeserializer extends JsonDeserializer<Country> {
+    @Override
+    public Country deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+      try {
+        if (jp != null && jp.getTextLength() > 0) {
+          return Country.fromIsoCode(jp.getText());
+        } else {
+          return Country.UNKNOWN; // none provided
+        }
+      } catch (Exception e) {
+        throw new IOException("Unable to deserialize country from provided value (not an ISO 2 character?): "
+          + jp.getText());
+      }
+    }
+  }
+
+
+}

registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java

@@ -0,0 +1,88 @@
+package org.gbif.registry.collections.sync;
+
+import lombok.Data;
+import retrofit2.Call;
+import retrofit2.Retrofit;
+import retrofit2.converter.jackson.JacksonConverterFactory;
+import retrofit2.http.GET;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Lightweight IndexHerbariorum client.
+ */
+class IndexHerbariorum {
+
+  static List<IndexHerbariorum.Institution> institutions() throws IOException {
+    return newClient().listInstitutions().execute().body().getData();
+  }
+
+
+  private static API newClient() {
+    Retrofit retrofit = new Retrofit.Builder()
+      //.baseUrl("http://sweetgum.nybg.org/science/api/v1/") // TODO: Parameterize
+      .baseUrl("http://labs.gbif.org/") // go easy on them...
+      .addConverterFactory(JacksonConverterFactory.create())
+      .build();
+    return retrofit.create(API.class);
+  }
+
+  private interface API {
+    @GET("institutions")
+    Call<InstitutionWrapper> listInstitutions();
+  }
+
+  @Data
+  private static class InstitutionWrapper {
+    private Meta meta;
+    private List<Institution> data;
+
+    @Data
+    private static class Meta {
+      private int hits;
+      private int code;
+    }
+  }
+
+
+  @Data
+  static class Institution {
+    private String irn;
+    private String organization;
+    private String code;
+    private String division;
+    private String department;
+    private long specimenTotal;
+    private Address address;
+    private Contact contact;
+    private Location location;
+    private String dateModified;
+
+    @Data
+    static class Address {
+      private String physicalStreet;
+      private String physicalCity;
+      private String physicalState;
+      private String physicalZipCode;
+      private String physicalCountry;
+      private String postalStreet;
+      private String postalCity;
+      private String postalState;
+      private String postalZipCode;
+      private String postalCountry;
+    }
+
+    @Data
+    static class Contact {
+      private String phone;
+      private String email;
+      private String webUrl;
+    }
+
+    @Data
+    static class Location {
+      private Double lat;
+      private Double lon;
+    }
+  }
+}