Skip to content

Commit

Permalink
All destinations (#152)
Browse files Browse the repository at this point in the history
* "Remote Connection" button and model

* share link is in working state

* works with separate account

* request works unauth

* add home router

* start of adding other destinations

* add template

* connection save message

* typo

* support different destination

* start of adding other destinations

* add template

* support different destination

* destination is immediately created

* start of adding other destinations

* add template

* support different destination

* start of adding other destinations

* support different destination

* requested changes

* start of adding other destinations

* add template

* support different destination

* start of adding other destinations

* support different destination

* start of adding other destinations

* add template

* start of adding other destinations

* support different destination

* add textarea type

* auto-parse form

* changes

---------

Co-authored-by: poundifdef <jay@scratchdata.com>
  • Loading branch information
breadchris and poundifdef committed Apr 10, 2024
1 parent 415c97b commit 18cdcf2
Show file tree
Hide file tree
Showing 12 changed files with 327 additions and 126 deletions.
1 change: 1 addition & 0 deletions go.mod
Expand Up @@ -96,6 +96,7 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gorilla/csrf v1.7.2 // indirect
github.com/gorilla/schema v1.3.0 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.2.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Expand Up @@ -200,6 +200,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh
github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
github.com/gorilla/csrf v1.7.2 h1:oTUjx0vyf2T+wkrx09Trsev1TE+/EbDAeHtSTbtC2eI=
github.com/gorilla/csrf v1.7.2/go.mod h1:F1Fj3KG23WYHE6gozCmBAezKookxbIvUJT+121wTuLk=
github.com/gorilla/schema v1.3.0 h1:rbciOzXAx3IB8stEFnfTwO3sYa6EWlQk79XdyustPDA=
github.com/gorilla/schema v1.3.0/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY=
Expand Down
15 changes: 8 additions & 7 deletions pkg/destinations/bigquery/bigquery.go
@@ -1,23 +1,24 @@
package bigquery

import (
"cloud.google.com/go/bigquery"
"context"

"cloud.google.com/go/bigquery"
"github.com/rs/zerolog/log"
"github.com/scratchdata/scratchdata/pkg/util"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
)

type BigQueryServer struct {
CredentialsJsonString string `mapstructure:"credentials_json"`
Location string `mapstructure:"location"`
// this will be implemented during runtime
CredentialsJsonString string `mapstructure:"credentials_json" schema:"credentials_json" form:"label:Credentials JSON String,type:textarea"`
Location string `mapstructure:"location" schema:"location" form:"label:Location,type:text"`

GCSBucketName string `mapstructure:"gcs_bucket_name"`
GCSFilePrefix string `mapstructure:"gcs_file_prefix"`
// this will be implemented during runtime
GCSBucketName string `mapstructure:"gcs_bucket_name" schema:"gcs_bucket_name" form:"label:GCS Bucket Name,type:text"`
GCSFilePrefix string `mapstructure:"gcs_file_prefix" schema:"gcs_file_prefix" form:"label:GCS File Prefix,type:text"`

DeleteFromGCS bool `mapstructure:"delete_from_gcs"`
DeleteFromGCS bool `mapstructure:"delete_from_gcs" schema:"delete_from_gcs" form:"label:Delete From GCS,type:bool"`

Credentials *google.Credentials
conn *bigquery.Client
Expand Down
16 changes: 8 additions & 8 deletions pkg/destinations/clickhouse/clickhouse.go
Expand Up @@ -18,14 +18,14 @@ import (
)

type ClickhouseServer struct {
Host string `mapstructure:"host"`
HTTPProtocol string `mapstructure:"http_protocol"`
HTTPPort int `mapstructure:"http_port"`
TCPPort int `mapstructure:"tcp_port"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
TLS bool `mapstructure:"tls"`
Host string `mapstructure:"host" schema:"host" form:"label:Host,type:text"`
Username string `mapstructure:"username" schema:"username" form:"label:Username,type:text"`
Password string `mapstructure:"password" schema:"password" form:"label:Password,type:password"`
Database string `mapstructure:"database" schema:"database" form:"label:Database Name,type:text,default:default"`
HTTPProtocol string `mapstructure:"http_protocol" schema:"http_protocol" form:"label:HTTP Protocol,type:text,default:https"`
HTTPPort int `mapstructure:"http_port" schema:"http_port" form:"label:HTTP Port,type:number,default:8443"`
TCPPort int `mapstructure:"tcp_port" schema:"tcp_port" form:"label:TCP Port,type:number,default:9440"`
TLS bool `mapstructure:"tls" schema:"tls" form:"label:TLS,type:bool,default:true"`

StoragePolicy string `mapstructure:"storage_policy"`

Expand Down
30 changes: 30 additions & 0 deletions pkg/destinations/config.go
@@ -0,0 +1,30 @@
package destinations

import (
"github.com/scratchdata/scratchdata/pkg/destinations/bigquery"
"github.com/scratchdata/scratchdata/pkg/destinations/clickhouse"
"github.com/scratchdata/scratchdata/pkg/destinations/duckdb"
"github.com/scratchdata/scratchdata/pkg/destinations/redshift"
)

var ViewConfig = map[string]struct {
Type any
Display string
}{
"duckdb": {
Type: duckdb.DuckDBServer{},
Display: "DuckDB",
},
"redshift": {
Type: redshift.RedshiftServer{},
Display: "Redshift",
},
"bigquery": {
Type: bigquery.BigQueryServer{},
Display: "BigQuery",
},
"clickhouse": {
Type: clickhouse.ClickhouseServer{},
Display: "Clickhouse",
},
}
16 changes: 13 additions & 3 deletions pkg/destinations/duckdb/duckdb.go
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"time"

"github.com/rs/zerolog/log"
Expand All @@ -17,8 +18,8 @@ import (
)

type DuckDBServer struct {
Database string `mapstructure:"database"`
Token string `mapstructure:"token"`
Database string `mapstructure:"database" schema:"database" form:"label:Database Name,type:text"`
Token string `mapstructure:"token" schema:"token" form:"label:MotherDuck Token,type:password"`

File string `mapstructure:"file"`

Expand Down Expand Up @@ -55,7 +56,16 @@ func openDB(s *DuckDBServer) (*sql.DB, error) {
}
connectionString = s.File
} else if s.Database != "" && s.Token != "" {
connectionString = "md:" + s.Database + "?motherduck_token=" + s.Token
if strings.Contains(strings.ToLower(s.Database), "saas_mode") {
return nil, errors.New("db cannot be named saas_mode")
}
if strings.Contains(strings.ToLower(s.Token), "saas_mode") {
return nil, errors.New("token cannot be named saas_mode")
}

connectionString = "md:" + s.Database + "?motherduck_saas_mode=true&motherduck_token=" + s.Token

// connectionString = "md:" + s.Database + "?motherduck_token=" + s.Token
} else {
return nil, errors.New("Must specify DuckDB connection type: in memory, file, or MotherDuck credentials")
}
Expand Down
25 changes: 13 additions & 12 deletions pkg/destinations/redshift/redshift.go
Expand Up @@ -3,27 +3,28 @@ package redshift
import (
"database/sql"
"fmt"

"github.com/rs/zerolog/log"
"github.com/scratchdata/scratchdata/pkg/util"

_ "github.com/lib/pq"
)

type RedshiftServer struct {
Host string `mapstructure:"redshift_host"`
Port int `mapstructure:"redshift_port"`
Username string `mapstructure:"redshift_user"`
Password string `mapstructure:"redshift_password"`
Database string `mapstructure:"redshift_database"`
Schema string `mapstructure:"redshift_schema"`
Host string `mapstructure:"redshift_host" schema:"redshift_host" form:"label:Host,type:text"`
Port int `mapstructure:"redshift_port" schema:"redshift_port" form:"label:Port,type:number,default:5439"`
Database string `mapstructure:"redshift_database" schema:"redshift_database" form:"label:Database,type:text"`
Username string `mapstructure:"redshift_user" schema:"redshift_user" form:"label:User,type:text"`
Password string `mapstructure:"redshift_password" schema:"redshift_password" form:"label:Password,type:password"`
Schema string `mapstructure:"redshift_schema" schema:"redshift_schema" form:"label:Schema,type:text,default:public"`

S3Region string `mapstructure:"s3_region"`
S3AccessKeyId string `mapstructure:"s3_access_key_id"`
S3SecretAccessKey string `mapstructure:"s3_secret_access_key"`
S3Bucket string `mapstructure:"s3_bucket"`
S3FilePrefix string `mapstructure:"s3_file_prefix"`
S3Region string `mapstructure:"s3_region" schema:"s3_region" form:"label:S3 Region,type:text"`
S3AccessKeyId string `mapstructure:"s3_access_key_id" schema:"s3_access_key_id" form:"label:S3 Access Key ID,type:text"`
S3SecretAccessKey string `mapstructure:"s3_secret_access_key" schema:"s3_secret_access_key" form:"label:S3 Secret Access Key,type:password"`
S3Bucket string `mapstructure:"s3_bucket" schema:"s3_bucket" form:"label:S3 Bucket,type:text"`
S3FilePrefix string `mapstructure:"s3_file_prefix" schema:"s3_file_prefix" form:"label:S3 File Prefix,type:text"`

DeleteFromS3 bool `mapstructure:"delete_from_s3"`
DeleteFromS3 bool `mapstructure:"delete_from_s3" schema:"delete_from_s3" form:"label:Delete From S3,type:bool"`
conn *sql.DB
}

Expand Down
57 changes: 57 additions & 0 deletions pkg/util/forms.go
@@ -0,0 +1,57 @@
package util

import (
"reflect"
"strings"
)

type Form struct {
Name string
Type string
Label string
Default string
}

// TODO breadchris return error if form tag is malformed
func ConvertToForms(server any) []Form {
var forms []Form

serverValue := reflect.ValueOf(server)
serverType := serverValue.Type()

for i := 0; i < serverValue.NumField(); i++ {
fieldType := serverType.Field(i)

form := fieldType.Tag.Get("form")
schema := fieldType.Tag.Get("schema")

parts := strings.Split(schema, ",")
schemaName := parts[0]

parts = strings.Split(form, ",")
formLabel := ""
formDefault := ""
formType := ""
for _, part := range parts {
if strings.HasPrefix(part, "label:") {
formLabel = strings.TrimPrefix(part, "label:")
}
if strings.HasPrefix(part, "default:") {
formDefault = strings.TrimPrefix(part, "default:")
}
if strings.HasPrefix(part, "type:") {
formType = strings.TrimPrefix(part, "type:")
}
}

if formType != "" {
forms = append(forms, Form{
Name: schemaName,
Type: formType,
Label: formLabel,
Default: formDefault,
})
}
}
return forms
}

0 comments on commit 18cdcf2

Please sign in to comment.