Merge pull request #8 from Recidiviz/dan/ssl-jdbc

Serve HTTPS connections to enable JDBC connection

README.md

@@ -188,6 +188,27 @@ read_client = bigquery_storage.BigQueryReadClient(client_options=client_options)
 result = client.query(sql).to_dataframe(bqstorage_client=read_client)
 ``` 
 
+## How to connect from JDBC
+Data exploration tools such as [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) or [DBeaver](https://dbeaver.com/) are immensely valuable during development of projects using the emulator.
+
+To connect the BigQuery Emulator to these tools from the JDBC, specify the following connection URL with your project ID:
+
+```
+jdbc:bigquery://https://localhost:9050;ProjectId=EMULATOR_PROJECT_ID;
+```
+
+The following options must also be specified:
+
+|Option| Value                   |Description|
+|--|-------------------------|--|
+|`RootURL`| `https://localhost:9070` |This overwrites the URL used in the JDBC driver|
+|`SSLTrustStore`|`/ssl/truststore.jks`| This must point to the JKS TrustStore that contains the SSL certificates for both the BigQuery Emulator, as well as `oauth2.googleapis.com`. A bundled truststore is located in the repository at `ssl/truststore.jks`. If using the Docker distribution, a volume can be created to access this data.|
+|`SSLTrustStorePwd`|`test@123`|This is the password for the trust store. The default password for the bundled store is `test@123`.|
+
+Known limitation: There is no documented method to bypass the JDBC OAuth flow, so a valid GCP OAuth credential is required to connect to the emulator.
+See `Configuring Authentication` in [this document](https://storage.googleapis.com/simba-bq-release/jdbc/Simba%20Google%20BigQuery%20JDBC%20Connector%20Install%20and%20Configuration%20Guide_1.5.2.1005.pdf) for a list of valid authentication methods.
+Exercise caution when using this feature with untrusted distributions of the BigQuery Emulator, as the OAuth Bearer token will be sent to the emulator server.
+
 # Synopsis
 
 If you use the Go language as a BigQuery client, you can launch the BigQuery emulator on the same process as the testing process.  

cmd/bigquery-emulator/main.go

@@ -18,6 +18,7 @@ type option struct {
 	Project      string           `description:"specify the project name" long:"project"`
 	Dataset      string           `description:"specify the dataset name" long:"dataset"`
 	HTTPPort     uint16           `description:"specify the http port number. this port used by bigquery api" long:"port" default:"9050"`
+	HTTPSPort    uint16           `description:"specify the https port number. this port can be used by bigquery api or for JDBC driver connections" long:"https-port" default:"9070"`
 	GRPCPort     uint16           `description:"specify the grpc port number. this port used by bigquery storage api" long:"grpc-port" default:"9060"`
 	LogLevel     server.LogLevel  `description:"specify the log level (debug/info/warn/error)" long:"log-level" default:"error"`
 	LogFormat    server.LogFormat `description:"specify the log format (console/json)" long:"log-format" default:"console"`
@@ -126,10 +127,12 @@ func runServer(args []string, opt option) error {
 	done := make(chan error)
 	go func() {
 		httpAddr := fmt.Sprintf("0.0.0.0:%d", opt.HTTPPort)
+		httpsAddr := fmt.Sprintf("0.0.0.0:%d", opt.HTTPSPort)
 		grpcAddr := fmt.Sprintf("0.0.0.0:%d", opt.GRPCPort)
 		fmt.Fprintf(os.Stdout, "[bigquery-emulator] REST server listening at %s\n", httpAddr)
+		fmt.Fprintf(os.Stdout, "[bigquery-emulator] REST HTTPS server listening at %s\n", httpsAddr)
 		fmt.Fprintf(os.Stdout, "[bigquery-emulator] gRPC server listening at %s\n", grpcAddr)
-		done <- bqServer.Serve(ctx, httpAddr, grpcAddr)
+		done <- bqServer.Serve(ctx, httpAddr, httpsAddr, grpcAddr)
 	}()
 
 	select {

server/server.go

@@ -208,7 +208,7 @@ func (s *Server) Load(sources ...Source) error {
 	return nil
 }
 
-func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error {
+func (s *Server) Serve(ctx context.Context, httpAddr, httpsAddr, grpcAddr string) error {
 	httpServer := &http.Server{
 		Handler:      s.Handler,
 		Addr:         httpAddr,
@@ -217,6 +217,13 @@ func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error {
 	}
 	s.httpServer = httpServer
 
+	httpsServer := &http.Server{
+		Handler:      s.Handler,
+		Addr:         httpsAddr,
+		WriteTimeout: 15 * time.Second,
+		ReadTimeout:  15 * time.Second,
+	}
+
 	grpcServer := grpc.NewServer()
 	registerStorageServer(grpcServer, s)
 	s.grpcServer = grpcServer
@@ -233,6 +240,7 @@ func (s *Server) Serve(ctx context.Context, httpAddr, grpcAddr string) error {
 	var eg errgroup.Group
 	eg.Go(func() error { return grpcServer.Serve(grpcListener) })
 	eg.Go(func() error { return httpServer.Serve(httpListener) })
+	eg.Go(func() error { return httpsServer.ListenAndServeTLS("server.crt", "server.key") })
 	return eg.Wait()
 }
 

ssl/README.md

@@ -0,0 +1,31 @@
+# SSL Configuration
+## `server.crt`
+This is the SSL certificate used by the HTTPS server.
+
+```bash
+openssl req -new -x509 -sha256 -key server.key -out server.crt -days 3650
+```
+
+## `server.key`
+Server SSL Key
+
+```bash
+openssl genrsa -out server.key 204
+```
+
+
+## `truststore.jks`
+This Truststore can be used for connecting to the BigQuery emulator via JDBC.
+It contains the BigQuery Emulator server certificate and the `oauth2.googleapis.com` certificate.
+
+```bash
+keytool -import -alias bigquery-emulator-localhost \
+  -file server.crt -keystore truststore.jks -storepass "test@123"
+
+keytool -import -alias oauth -file upload.video.google.com.cer \
+  -keystore truststore.jks -storepass "test@123"
+```
+
+
+## `upload.video.google.com.crt`
+Public certificate for `oauth2.googleapis.com`

ssl/server.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDCTCCAfGgAwIBAgIUI3KcuYLsMbY8uAktUn7rAp5W/AAwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDMwNTE2MDE1NloXDTM0MDMw
+MzE2MDE1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAwlAVNzms9VJeThXOTocyPvbQ6TAJNZPpq0C35obpxs8H
+6tfvgGrkqwlH91b56dAK/OBm4k4VzHrbS3KDm6PFpe84T8R2bNLm6CXc6oftC7Ud
+mOF3dNvaBw0M+NyjXXEGISKEWXkYkF2luo7q9MQLgQWU16zKG5ktfjvMuCL9988y
+xjmSOjO5Icn6MQ/AVU7vlxYMyC+Go82xsiq+QNaO1mJSvZ3QZFwq0T+Ek5e5cwYg
+n2vhaqs1ZT7fVfTIU0vqzAH3AbTRpttNxuQfNcTmlu3iMlLS5bdYv7dWRR610SvU
+4UbrPwSWOkA5Oj6J/h9/mDWpBY1I85sYGdhOtNAs9wIDAQABo1MwUTAdBgNVHQ4E
+FgQUAgU21Hg8bp9lBbZrUwtI8o5pcbUwHwYDVR0jBBgwFoAUAgU21Hg8bp9lBbZr
+UwtI8o5pcbUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAjd7R
+60Fruosar4JskceNA2eT7YzQa57docQRq3oakVv7YkKqEpJjjBNO4apkCYD+JYYs
+HHL239j3VcFWv7qtkOYHcwlVtdr1a/zN4eQ7762aH/YGSqT9G7J7fqSfbK1DF80o
+hrj2qP3FQLHyeENNTfF4EAmpq6/0PnbIfDXjbPx+PxZ7XJJ4j667tvYkRupzPyx0
+uA1hgv0xWMhwIGItKhUr5BTApo3SrmeYGAQlWG+KboTQJJJ0u2aF5+aa1fw2COH0
+WeRDJzw28MWKn0GCBml4liIib4R4AfyEepyVaRvHxpC/ArwQyqI7GmSwIX78jWQg
+RO0t6zfvslGm8AwGbA==
+-----END CERTIFICATE-----

ssl/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDCUBU3Oaz1Ul5O
+Fc5OhzI+9tDpMAk1k+mrQLfmhunGzwfq1++AauSrCUf3Vvnp0Ar84GbiThXMettL
+coObo8Wl7zhPxHZs0uboJdzqh+0LtR2Y4Xd029oHDQz43KNdcQYhIoRZeRiQXaW6
+jur0xAuBBZTXrMobmS1+O8y4Iv33zzLGOZI6M7khyfoxD8BVTu+XFgzIL4ajzbGy
+Kr5A1o7WYlK9ndBkXCrRP4STl7lzBiCfa+FqqzVlPt9V9MhTS+rMAfcBtNGm203G
+5B81xOaW7eIyUtLlt1i/t1ZFHrXRK9ThRus/BJY6QDk6Pon+H3+YNakFjUjzmxgZ
+2E600Cz3AgMBAAECggEABRd+zCkBaexJTFfbRXOFfYuPgzd0W6hklyLlrsTE8NC8
+BGC5he6cegmO4+GuQCeTd3eUj+7mjf/U1Jqg7ZMnurwUswJoQrHEGzviw203E6Ig
+rxY5ayEtCsJCH23+BJTRnvdTQIT2f9H0Zc8Y7mZfZszJTwgtbku1RXmLrnI8Ssy3
+ZcPlzmYCVxz4lpZJGCFbcv/GHP+emRn4d3BgtLwuPNXld6li98HwIXSYvdUzx3sY
+R4sf6r9OPC4ugr11s4GmhaAh4mb52DjIsM3U7eI1aM5zZNW66qLcpbfe2Mtx0FwG
+ckKekWlFJ20PFxfKOjhNr/FOEZPmKq2PqWwtvKZqAQKBgQD4sUX1P8JimFuFazet
+H9NB0mEYMMXncb4gXjOHAlznFaozyt61rPHpeW7DHQvaE5Tj2YnjhqbDCXwx39XF
+nPYvAUHSzeMMtr5frbKCMPwhqWY8zbVyN9k99afhn45AQEzdxjovU7ra43jDiZ3E
+A8EKGJMfX5cHUPi99l+b5XYC/wKBgQDIBcCYqS2meAsfLJIo6TE1Eb3utlv3u+n3
+rUfjzHRxJEbiGEimeKvcqctCsOPhdRN1nza+NTn3Aak7onLSEkHz8xmnzLwRYWkd
+9gkB1NA6il7e8MUqj/zR9R0V8Fpn4X3AgQ1n1wnyF6RNUcnrC3t9z0nKaT8KMfqm
+NywacB/uCQKBgHRD9G/e2hLcJdVvNb1TZqGcKFZ13uAANiHNyIsy2JoUb9j83DfV
+XdeINPc06iiPBQms1yEu/2GpWo26lqdnZVS+YraaGK8F9GSowQ7KteVK3AhLJ6v5
+Xi/wXAdIX+m75fO8y9D6dR4GQJwyBzbvhl0g9g8x0lrSSSgOOc6ZbR2jAoGADXc1
+rBFXnWlfdk3N2Ss5wNTc8IEeV+MysRXdTRyqiiNjEj2IRozBJS3ZHThDXx4+nSCG
+0u9KY7Kc7gLVsAbCoeWvyHgkAReJuBakLJdjHU3LbT7QRzlCT/AscRmpPG0Vxivp
+9x+m78FfskTbTxOK0MFvukyzjcAnm6EDOxRpUeECgYAhXQOkTCF8iEeur/4GBkv/
+eGl4cAF4bE/C43JpsMPFkGH6O/Q/0HKIoFPQKLQixMDxcBDXEtJIJM4dem8/WLfX
+G9i2792n0bqo2wEyUFoD7+B+gzf7b0N8TgSNFqXdKFNS2/yt1QQEdvONnSxeXLiH
+AOn6ZsVi0zeUPGGmMJ6KVw==
+-----END PRIVATE KEY-----

ssl/upload.video.google.com.crt

@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGETCCBPmgAwIBAgIRAJGz+SndLQSMEp5fHZIwmzMwDQYJKoZIhvcNAQELBQAw
+RjELMAkGA1UEBhMCVVMxIjAgBgNVBAoTGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBM
+TEMxEzARBgNVBAMTCkdUUyBDQSAxQzMwHhcNMjQwMjA1MDgxOTE5WhcNMjQwNDI5
+MDgxOTE4WjAiMSAwHgYDVQQDExd1cGxvYWQudmlkZW8uZ29vZ2xlLmNvbTBZMBMG
+ByqGSM49AgEGCCqGSM49AwEHA0IABDAcQovXhW8dpwu0MvmE8X5qY1c+YpM6SMnm
+AW9T83bHXtEMjErp6Tb1mPXIONensRnOFM/XBeKX9bDrUnX1qtujggPnMIID4zAO
+BgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIw
+ADAdBgNVHQ4EFgQUn1hCQnptHBb/wohvDF53QccevFowHwYDVR0jBBgwFoAUinR/
+r4XN7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhto
+dHRwOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9w
+a2kuZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwggGYBgNVHREEggGPMIIBi4IX
+dXBsb2FkLnZpZGVvLmdvb2dsZS5jb22CFCouY2xpZW50cy5nb29nbGUuY29tghEq
+LmRvY3MuZ29vZ2xlLmNvbYISKi5kcml2ZS5nb29nbGUuY29tghMqLmdkYXRhLnlv
+dXR1YmUuY29tghAqLmdvb2dsZWFwaXMuY29tghMqLnBob3Rvcy5nb29nbGUuY29t
+ghcqLnlvdXR1YmUtM3JkLXBhcnR5LmNvbYIRdXBsb2FkLmdvb2dsZS5jb22CEyou
+dXBsb2FkLmdvb2dsZS5jb22CEnVwbG9hZC55b3V0dWJlLmNvbYIUKi51cGxvYWQu
+eW91dHViZS5jb22CH3VwbG9hZHMuc3RhZ2UuZ2RhdGEueW91dHViZS5jb22CFWJn
+LWNhbGwtZG9uYXRpb24uZ29vZ4IbYmctY2FsbC1kb25hdGlvbi1hbHBoYS5nb29n
+ghxiZy1jYWxsLWRvbmF0aW9uLWNhbmFyeS5nb29nghliZy1jYWxsLWRvbmF0aW9u
+LWRldi5nb29nMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYD
+VR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL2ZWSnhi
+Vi1LdG1rLmNybDCCAQMGCisGAQQB1nkCBAIEgfQEgfEA7wB2AHb/iD8KtvuVUcJh
+zPWHujS0pM27KdxoQgqf5mdMWjp0AAABjXiQoKYAAAQDAEcwRQIhANF3yjIlxIXq
+1LhdAylbh2Uz6biXTUcuQKpC72DR04E6AiBK5iFHAGm6O76mHNB0xXPrmC9c5M+E
+JqLX5qQ0TUbD4wB1AEiw42vapkc0D+VqAvqdMOscUgHLVt0sgdm7v6s52IRzAAAB
+jXiQoJgAAAQDAEYwRAIgaRH4gdjAbABDzaaPMQMC4nRPd2b70j2BzhuZ6ucvqgwC
+IHNQeCXokQcSm9dr6Ib8WunxJ50AZZgxMSHrKUWodk8sMA0GCSqGSIb3DQEBCwUA
+A4IBAQAWz8k1csY9p67eyZ+bCkHB1rGvBm8Pmi2RVbh29csbP/wwTWFDGQ9DoCfs
+q5vR+bEc50vXbxhZZEqR4pGU/y9aj1F21yCymwmrsxf48/bm0G2jGIeEhYzGGVUN
+9ap8+V2fCFKnK0LHr8szEj/eYJ38tuHgnmyEREiUtDTMK7P8bHwbDSbNM58AWxjB
+ljS0e+1jzyZW3HIroZniu481SzeDsP/CgakAhEAN6tWajq1gzkkIXAuLF9SoVAr+
+njOatrO1yIGerDuDSvIc03s71uIAv5SO1YWM7MJGgsfHIBEKuZXGjV7Z5t8dc+1D
+5uWuWPzalEV1XVbYeOwYDnDXrq3u
+-----END CERTIFICATE-----