fix: use common chart, prepare for gcp2

.circleci/config.yml

@@ -111,4 +111,6 @@ workflows:
             - build
           filters:
             branches:
-              only: master
+              only:
+                - master
+                - prepare-gcp2

.gitignore

@@ -43,3 +43,5 @@ src/main/resources/stops/agency.txt
 
 local_config/
 node_modules
+
+.terraform

helm/tiamat-gcp2/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: https://entur.github.io/helm-charts
+  version: 1.1.0
+digest: sha256:24dad2384359deaa8d9e7ec582eb58aef4980abedfce52a84d9c3c6eea3b8eb0
+generated: "2022-08-09T09:46:14.789654282+02:00"

helm/tiamat-gcp2/Chart.yaml

@@ -0,0 +1,17 @@
+apiVersion: v2
+appVersion: "0.1.99.2"
+description: NSR (National Stoppalce Register)
+name: tiamat
+version: 0.1.0
+keywords:
+- kakka
+sources:
+- https://github.com/entur/tiamat
+maintainers:
+- name: 'team-ror'
+  email: team.rutedata@entur.org
+engine: gotpl
+dependencies:
+  - name: common
+    version: 1.1.0
+    repository: "https://entur.github.io/helm-charts"

helm/tiamat-gcp2/templates/configmap.yaml

@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tiamat-application
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: release-name
+    shortname: tiamatno
+    team: ror
+data:
+  application.properties: |-
+    server.port={{ .Values.common.service.internalPort }}
+    nearbyStopsWithSameTypeFinder.limits={railStation:'200',airport:'3000'}
+    tariffzoneLookupService.resetReferences=true
+    publicationDeliveryStreamingOutput.validateAgainstSchema=true
+    publicationDeliveryUnmarshaller.validateAgainstSchema=true
+    netex.import.enabled.types=MERGE
+    changelog.publish.enabled=true
+    spring.activemq.broker-url=tcp://activemq.default:61616?jms.redeliveryPolicy.maximumRedeliveries=0
+    netex.id.valid.prefix.list={TopographicPlace:{'KVE','WOF','OSM','ENT','LAN'},TariffZone:{'*'},FareZone:{'*'},GroupOfTariffZones:{'*'}}
+    netex.profile.version=1.12:NO-NeTEx-stops:1.4
+
+    # GCS properties
+    blobstore.gcs.bucket.name={{.Values.configMap.gcsBucketName}}
+    blobstore.gcs.blob.path=export
+    blobstore.gcs.project.id={{.Values.configMap.blobstoreProjectId}}
+
+    # Database properties
+    spring.jpa.properties.javax.persistence.query.timeout=15000
+    spring.jpa.properties.hibernate.dialect=org.hibernate.spatial.dialect.postgis.PostgisDialect
+    spring.jpa.properties.hibernate.cache.region.factory_class=org.rutebanken.tiamat.hazelcast.TiamatHazelcastCacheRegionFactory
+    spring.jpa.properties.hibernate.cache.use_second_level_cache=true
+    spring.jpa.properties.hibernate.cache.use_minimal_puts=true
+    spring.jpa.properties.hibernate.cache.use_query_cache=true
+    spring.jpa.hibernate.ddl-auto=none
+    spring.jpa.hibernate.use-new-id-generator-mappings=true
+    spring.jpa.open-in-view=false
+    spring.jpa.database=POSTGRESQL
+    spring.jpa.show-sql=false
+    spring.database.driverClassName=org.postgresql.Driver
+    spring.datasource.hikari.maximumPoolSize=30
+    spring.datasource.url=jdbc:postgresql://localhost:5432/{{ .Values.tiamat.database }}
+    spring.datasource.username=tiamat
+    spring.datasource.platform=postgres
+    spring.datasource.password=${PGPASSWORD}
+    graphql.query.max.depth =100
+
+
+    #flyway
+    spring.flyway.table =schema_version
+
+
+
+    # Security config with auth0
+
+    authorization.enabled=true
+
+    #OAuth2 Resource Server
+    spring.security.oauth2.resourceserver.jwt.issuer-uri={{ .Values.auth0.ror.url }}/
+    tiamat.oauth2.resourceserver.auth0.ror.jwt.audience={{ .Values.auth0.ror.audience }}
+    tiamat.oauth2.resourceserver.auth0.ror.claim.namespace=https://ror.entur.io/
+
+
+    # Logging Properties
+    logging.level.org.rutebanken.tiamat=INFO
+    logging.level.org.rutebanken.tiamat.repository=DEBUG
+    logging.level.org.rutebanken.tiamat.importer.finder=INFO
+    logging.level.org.rutebanken.tiamat.dtoassembling.disassembler.StopPlaceSearchDisassembler=WARN
+    logging.level.org.rutebanken.tiamat.rest.netex.publicationdelivery.PublicationDeliveryUnmarshaller=INFO
+    logging.level.org.rutebanken.tiamat.rest.netex.publicationdelivery=INFO
+    logging.level.org.rutebanken.tiamat.rest.netex.publicationdelivery.PublicationDeliveryStreamingOutput=INFO
+    logging.level.org.springframework.orm.hibernate4.support=WARN
+    logging.level.org.hibernate.SQL=WARN
+    logging.level.org.hibernate.tool.hbm2ddl=INFO
+    logging.level.org.hibernate.type=WARN
+    logging.level.org.hibernate.id.enhanced=INFO
+
+    #Profile
+    spring.profiles.active=gcs-blobstore,google-pubsub
+
+    # pubsub
+    spring.cloud.gcp.project-id={{ .Values.configMap.pubsubProjectId }}
+    changelog.topic.name={{ .Values.configMap.pubsubTopicName }}
+

helm/tiamat-gcp2/values-kub-ent-dev-nordic.yaml

@@ -0,0 +1,14 @@
+common:
+  app: tiamat-nordic
+  env: dev
+
+configMap:
+  blobstoreProjectId: entur-ror-dev
+  pubsubProjectId: entur-ror-dev
+  pubsubTopicName: ror.tiamat-nordic.changelog
+  gcsBucketName: tiamat-nordic-dev
+
+auth0:
+  ror:
+    url: https://ror-entur-dev.eu.auth0.com
+    audience: https://ror.api.dev.entur.io

helm/tiamat-gcp2/values-kub-ent-dev.yaml

@@ -0,0 +1,2 @@
+common:
+  env: dev

helm/tiamat-gcp2/values-kub-ent-prd.yaml

@@ -0,0 +1,2 @@
+common:
+  env: prd

helm/tiamat-gcp2/values-kub-ent-tst.yaml

@@ -0,0 +1,2 @@
+common:
+  env: tst

helm/tiamat-gcp2/values.yaml

@@ -0,0 +1,45 @@
+tiamat:
+  database: tiamat-ns
+common:
+  app: tiamat
+  shortname: tiamatno 
+  team: ror 
+  service:
+    internalPort: 8777
+  ingress:
+    host: stop-places-v1-nordic.dev.entur.io
+    trafficType: api
+  container:
+    image: ${artifact.metadata.image}
+    cpu: 1.2
+    memory: 7500
+    probes:
+      liveness:
+        path: /health/live
+      readiness:
+        path: /health/ready
+    envFrom:
+      - configMapRef:
+          name: tiamat-application
+    prometheus:
+      enabled: true
+      path: /health/scrape
+    volumeMounts:
+    - mountPath: /etc/application-config
+      name: application-config
+      readOnly: true
+    volumes:
+    - configMap:
+        defaultMode: 420
+        name: tiamat-application
+      name: application-config
+  postgres:
+    enabled: true
+  configmap:
+    data:
+      JAVA_OPTIONS: -server -Xss1024m -Xmx7g -Dspring.config.location=/etc/application-config/application.properties
+        -Dfile.encoding=UTF-8 -Drutebanken.kubernetes.enabled=true -Drutebanken.hazelcast.management.url=http://tiamat-hzmgmt/mancenter/
+        -Drutebanken.kubernetes.namespace=default -Dhazelcast.ignoreXxeProtectionFailures=true
+      TZ: Europe/Oslo
+    enabled: true
+    toEnv: true

src/main/java/org/rutebanken/tiamat/service/GcsBlobStoreService.java

@@ -16,6 +16,8 @@
 package org.rutebanken.tiamat.service;
 
 import com.google.cloud.storage.Storage;
+import com.google.cloud.http.HttpTransportOptions;
+import com.google.cloud.storage.StorageOptions;
 import org.rutebanken.helper.gcp.BlobStoreHelper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -30,23 +32,21 @@
 @Profile("gcs-blobstore")
 public class GcsBlobStoreService implements BlobStoreService {
 
+    private static final int CONNECT_AND_READ_TIMEOUT = 60000;
     private static final Logger logger = LoggerFactory.getLogger(GcsBlobStoreService.class);
 
     private final String bucketName;
 
     private final String blobPath;
-    private final String credentialPath;
     private final String projectId;
 
 
-    public GcsBlobStoreService(@Value("${blobstore.gcs.credential.path}") String credentialPath,
-                            @Value("${blobstore.gcs.bucket.name}") String bucketName,
+    public GcsBlobStoreService(@Value("${blobstore.gcs.bucket.name}") String bucketName,
                             @Value("${blobstore.gcs.blob.path}") String blobPath,
                             @Value("${blobstore.gcs.project.id}") String projectId) {
 
         this.bucketName = bucketName;
         this.blobPath = blobPath;
-        this.credentialPath = credentialPath;
         this.projectId = projectId;
     }
 
@@ -62,11 +62,19 @@ public void upload(String fileName, InputStream inputStream) {
     }
 
     public Storage getStorage() {
+        logger.info("Get storage for project {}", projectId);
+
         try {
-            logger.info("Get storage for project {}", projectId);
-            return BlobStoreHelper.getStorage(credentialPath, projectId);
+            HttpTransportOptions transportOptions = StorageOptions.getDefaultHttpTransportOptions();
+            transportOptions = transportOptions.toBuilder().setConnectTimeout(CONNECT_AND_READ_TIMEOUT).setReadTimeout(CONNECT_AND_READ_TIMEOUT)
+                    .build();
+
+            return StorageOptions.newBuilder()
+                    .setProjectId(projectId)
+                    .setTransportOptions(transportOptions)
+                    .build().getService();
         } catch (RuntimeException e) {
-            throw new RuntimeException("Error setting up BlobStore from blobstore.gcs.credential.path '" + credentialPath + "' and blobstore.gcs.project.id '" + projectId + "'", e);
+            throw new RuntimeException(e);
         }
     }
 

src/test/resources/application.properties

@@ -34,7 +34,6 @@ blobstore.gcs.blob.path=exports
 blobstore.gcs.bucket.name=tiamat-test
 
 # Replace with your gcload storage
-blobstore.gcs.credential.path=gcloud-storage.json
 blobstore.gcs.project.id=carbon-1287
 
 server.port=1888

terraform-nordic/.tool-versions

@@ -0,0 +1 @@
+terraform 0.13.2

terraform-nordic/backend.tf

@@ -0,0 +1,5 @@
+terraform {
+  backend "gcs" {
+    bucket = "ent-gcs-tfa-tiamatno"
+  }
+}

terraform-nordic/env/kub-ent-dev.tfvars

@@ -0,0 +1 @@
+env = "dev"

terraform-nordic/main.tf

@@ -0,0 +1,49 @@
+locals {
+  topic_name = "${module.init.app.owner}.${module.init.app.name}.changelog"
+}
+
+# Terraform configuration for tiamat
+module "init" {
+  source      = "github.com/entur/terraform-google-init//modules/init?ref=v0.2.1"
+  app_id      = "tiamatno"
+  environment = var.env
+}
+
+# https://github.com/entur/terraform-google-sql-db/tree/master/modules/postgresql#inputs
+module "postgres" {
+  source           = "github.com/entur/terraform-google-sql-db//modules/postgresql?ref=v0.1.2"
+  init             = module.init
+  generation       = 1
+  database_version = "POSTGRES_14"
+  databases        = ["tiamatno"]
+}
+
+# https://github.com/entur/terraform-google-cloud-storage/tree/master/modules/bucket#inputs
+module "cloud-storage" {
+  source = "github.com/entur/terraform-google-cloud-storage//modules/bucket?ref=v0.1.0"
+  init   = module.init
+}
+
+# Create folder in a bucket
+resource "google_storage_bucket_object" "content_folder" {
+  name    = "export/"
+  content = "Not really a directory, but it's empty."
+  bucket  = module.cloud-storage.cloud_storage_bucket.name
+}
+
+# Create pubsub topic
+resource "google_pubsub_topic" "changelog" {
+  name    = local.topic_name
+  project = module.init.app.project_id
+  labels  = module.init.labels
+}
+
+# Create pubsub subscription
+resource "google_pubsub_subscription" "changelog-subscription" {
+  name    = local.topic_name
+  topic   = google_pubsub_topic.changelog.name
+  project = module.init.app.project_id
+  labels  = module.init.labels
+}
+
+

terraform-nordic/variables.tf

@@ -0,0 +1,3 @@
+variable "env" {
+  description = "Environment. dev|tst|prd."
+}