Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
#159 Skeleton code for Index Herbariorum synchronization
- Loading branch information
1 parent
ef14d31
commit fde4f7e
Showing
6 changed files
with
433 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Registry Collections Synchronisation | ||
|
||
Provides synchronisation utilities for key repositories such as Index Hebrarirum. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,95 @@ | ||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | ||
<modelVersion>4.0.0</modelVersion> | ||
|
||
<parent> | ||
<groupId>org.gbif.registry</groupId> | ||
<artifactId>registry-motherpom</artifactId> | ||
<version>2.140-SNAPSHOT</version> | ||
</parent> | ||
|
||
<artifactId>registry-collections-sync</artifactId> | ||
<packaging>jar</packaging> | ||
|
||
<name>Registry Collections Sync</name> | ||
|
||
<distributionManagement> | ||
<site> | ||
<id>gh-pages</id> | ||
<url>http://gbif.github.io/registry/${project.artifactId}/</url> | ||
</site> | ||
</distributionManagement> | ||
|
||
<properties> | ||
<slf4j.version>1.7.29</slf4j.version> | ||
<jackson.version>2.9.4</jackson.version> | ||
<retrofit.version>2.5.0</retrofit.version> | ||
<okhttp.version>3.11.0</okhttp.version> | ||
<okio.version>2.0.0</okio.version> | ||
<lombok.version>1.18.10</lombok.version> | ||
</properties> | ||
|
||
<repositories> | ||
<repository> | ||
<id>gbif-all</id> | ||
<url>http://repository.gbif.org/content/groups/gbif</url> | ||
</repository> | ||
<repository> | ||
<id>gbif-thirdparty</id> | ||
<url>http://repository.gbif.org/content/repositories/thirdparty/</url> | ||
</repository> | ||
</repositories> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>org.gbif</groupId> | ||
<artifactId>gbif-api</artifactId> | ||
<exclusions> | ||
<exclusion> | ||
<groupId>org.codehaus.jackson</groupId> | ||
<artifactId>*</artifactId> | ||
</exclusion> | ||
</exclusions> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.fasterxml.jackson.core</groupId> | ||
<artifactId>jackson-annotations</artifactId> | ||
<version>${jackson.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.slf4j</groupId> | ||
<artifactId>slf4j-api</artifactId> | ||
<version>${slf4j.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.slf4j</groupId> | ||
<artifactId>slf4j-simple</artifactId> | ||
<version>${slf4j.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.squareup.retrofit2</groupId> | ||
<artifactId>retrofit</artifactId> | ||
<version>${retrofit.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.squareup.retrofit2</groupId> | ||
<artifactId>converter-jackson</artifactId> | ||
<version>${retrofit.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.squareup.okhttp3</groupId> | ||
<artifactId>okhttp</artifactId> | ||
<version>${okhttp.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.squareup.okio</groupId> | ||
<artifactId>okio</artifactId> | ||
<version>${okio.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>org.projectlombok</groupId> | ||
<artifactId>lombok</artifactId> | ||
<version>${lombok.version}</version> | ||
<scope>provided</scope> | ||
</dependency> | ||
</dependencies> | ||
</project> |
101 changes: 101 additions & 0 deletions
101
registry-collections-sync/src/main/java/org/gbif/registry/collections/sync/GRSciColl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
package org.gbif.registry.collections.sync; | ||
|
||
import com.fasterxml.jackson.core.JsonParser; | ||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import com.fasterxml.jackson.databind.DeserializationContext; | ||
import com.fasterxml.jackson.databind.JsonDeserializer; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.fasterxml.jackson.databind.module.SimpleModule; | ||
import org.gbif.api.model.collections.Collection; | ||
import org.gbif.api.model.collections.Institution; | ||
import org.gbif.api.model.common.paging.PagingResponse; | ||
import org.gbif.api.vocabulary.Country; | ||
import retrofit2.Call; | ||
import retrofit2.Retrofit; | ||
import retrofit2.converter.jackson.JacksonConverterFactory; | ||
import retrofit2.http.GET; | ||
import retrofit2.http.Query; | ||
import java.io.IOException; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
|
||
/** | ||
* A lightweight GRSciColl client. | ||
*/ | ||
public class GRSciColl { | ||
private static API api = newClient(); | ||
|
||
/** | ||
* Returns all institutions in GrSciCol. | ||
*/ | ||
static List<Institution> institutions() throws IOException { | ||
List<Institution> result = new ArrayList<>(); | ||
boolean endRecords = false; | ||
int offset = 0; | ||
while (!endRecords) { | ||
PagingResponse<Institution> response = api.listInstitutions(1000, offset).execute().body(); | ||
endRecords = response.isEndOfRecords(); | ||
offset += response.getLimit(); | ||
result.addAll(response.getResults()); | ||
} | ||
return result; | ||
} | ||
|
||
/** | ||
* Returns all institutions in GrSciCol. | ||
*/ | ||
static List<Collection> collections() throws IOException { | ||
List<Collection> result = new ArrayList<>(); | ||
boolean endRecords = false; | ||
int offset = 0; | ||
while (!endRecords) { | ||
PagingResponse<Collection> response = api.listCollections(1000, offset).execute().body(); | ||
endRecords = response.isEndOfRecords(); | ||
offset += response.getLimit(); | ||
result.addAll(response.getResults()); | ||
} | ||
return result; | ||
} | ||
|
||
private static API newClient() { | ||
ObjectMapper mapper = new ObjectMapper(); | ||
SimpleModule module = new SimpleModule(); | ||
module.addDeserializer(Country.class, new IsoDeserializer()); | ||
mapper.registerModule(module); | ||
|
||
Retrofit retrofit = new Retrofit.Builder() | ||
.baseUrl("http://api.gbif.org/v1/grscicoll/") // TODO parameterize | ||
.addConverterFactory(JacksonConverterFactory.create(mapper)) | ||
.build(); | ||
return retrofit.create(API.class); | ||
} | ||
|
||
private interface API { | ||
@GET("institution") | ||
Call<PagingResponse<Institution>> listInstitutions(@Query("limit") int limit, @Query("offset") int offset); | ||
|
||
@GET("collection") | ||
Call<PagingResponse<Collection>> listCollections(@Query("limit") int limit, @Query("offset") int offset); | ||
} | ||
|
||
/** | ||
* Adapter necessary for retrofit due to versioning. | ||
*/ | ||
private static class IsoDeserializer extends JsonDeserializer<Country> { | ||
@Override | ||
public Country deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { | ||
try { | ||
if (jp != null && jp.getTextLength() > 0) { | ||
return Country.fromIsoCode(jp.getText()); | ||
} else { | ||
return Country.UNKNOWN; // none provided | ||
} | ||
} catch (Exception e) { | ||
throw new IOException("Unable to deserialize country from provided value (not an ISO 2 character?): " | ||
+ jp.getText()); | ||
} | ||
} | ||
} | ||
|
||
|
||
} |
88 changes: 88 additions & 0 deletions
88
...y-collections-sync/src/main/java/org/gbif/registry/collections/sync/IndexHerbariorum.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
package org.gbif.registry.collections.sync; | ||
|
||
import lombok.Data; | ||
import retrofit2.Call; | ||
import retrofit2.Retrofit; | ||
import retrofit2.converter.jackson.JacksonConverterFactory; | ||
import retrofit2.http.GET; | ||
import java.io.IOException; | ||
import java.util.List; | ||
|
||
/** | ||
* Lightweight IndexHerbariorum client. | ||
*/ | ||
class IndexHerbariorum { | ||
|
||
static List<IndexHerbariorum.Institution> institutions() throws IOException { | ||
return newClient().listInstitutions().execute().body().getData(); | ||
} | ||
|
||
|
||
private static API newClient() { | ||
Retrofit retrofit = new Retrofit.Builder() | ||
//.baseUrl("http://sweetgum.nybg.org/science/api/v1/") // TODO: Parameterize | ||
.baseUrl("http://labs.gbif.org/") // go easy on them... | ||
.addConverterFactory(JacksonConverterFactory.create()) | ||
.build(); | ||
return retrofit.create(API.class); | ||
} | ||
|
||
private interface API { | ||
@GET("institutions") | ||
Call<InstitutionWrapper> listInstitutions(); | ||
} | ||
|
||
@Data | ||
private static class InstitutionWrapper { | ||
private Meta meta; | ||
private List<Institution> data; | ||
|
||
@Data | ||
private static class Meta { | ||
private int hits; | ||
private int code; | ||
} | ||
} | ||
|
||
|
||
@Data | ||
static class Institution { | ||
private String irn; | ||
private String organization; | ||
private String code; | ||
private String division; | ||
private String department; | ||
private long specimenTotal; | ||
private Address address; | ||
private Contact contact; | ||
private Location location; | ||
private String dateModified; | ||
|
||
@Data | ||
static class Address { | ||
private String physicalStreet; | ||
private String physicalCity; | ||
private String physicalState; | ||
private String physicalZipCode; | ||
private String physicalCountry; | ||
private String postalStreet; | ||
private String postalCity; | ||
private String postalState; | ||
private String postalZipCode; | ||
private String postalCountry; | ||
} | ||
|
||
@Data | ||
static class Contact { | ||
private String phone; | ||
private String email; | ||
private String webUrl; | ||
} | ||
|
||
@Data | ||
static class Location { | ||
private Double lat; | ||
private Double lon; | ||
} | ||
} | ||
} |
Oops, something went wrong.