Skip to content

Commit

Permalink
bigquery: use the proper table name for BQ destination
Browse files Browse the repository at this point in the history
We have two ways of saving data to BQ:

 1. client --(UDP)--> evenkitdbq --> BQ
 2. app --(Google API)--> BQ

In both cases, we should use the same table name convention (`scope_eventname`)

Change-Id: Id4677c20eb54ff1742a3f732ea0fac41153a3934
  • Loading branch information
elek committed Mar 6, 2024
1 parent e871f0a commit 6cb545e
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 9 deletions.
10 changes: 5 additions & 5 deletions bigquery/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,15 +184,15 @@ tagloop:
return nil
}

func TableName(event *pb.Event) string {
// TableName defines the naming convention between event and BQ table name.
func TableName(scope []string, name string) string {
var res []string
for _, scope := range event.Scope {
for _, scope := range scope {
res = append(res, nonSafeTableNameCharacters.ReplaceAllString(scope, "_"))
}
res = append(res, nonSafeTableNameCharacters.ReplaceAllString(event.Name, "_"))
res = append(res, nonSafeTableNameCharacters.ReplaceAllString(name, "_"))

name := strings.Join(res, "_")
all := multiUnderscore.ReplaceAllString(name, "_")
all := multiUnderscore.ReplaceAllString(strings.Join(res, "_"), "_")
all = strings.Trim(all, "_")
return all
}
Expand Down
7 changes: 4 additions & 3 deletions bigquery/destination.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,11 @@ func (b *BigQueryDestination) Submit(events ...*eventkit.Event) {
Value: t.Value,
})
}
if _, found := records[event.Name]; !found {
records[event.Name] = make([]*Record, 0)
tableName := TableName(event.Scope, event.Name)
if _, found := records[tableName]; !found {
records[tableName] = make([]*Record, 0)
}
records[event.Name] = append(records[event.Name], &Record{
records[tableName] = append(records[tableName], &Record{
Application: Application{
Name: b.appName,
Version: "0.0.1",
Expand Down
2 changes: 1 addition & 1 deletion eventkitd-bigquery/bigquery/sink.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (b *BigQuerySink) Receive(ctx context.Context, unparsed *listener.Packet, p
eventTime := correctedStart.Add(time.Duration(event.TimestampOffsetNs) * time.Nanosecond)
correction := correctedStart.Sub(packet.StartTimestamp.AsTime())

k := bigquery.TableName(event)
k := bigquery.TableName(event.Scope, event.Name)

records[k] = append(records[k], &bigquery.Record{
Application: bigquery.Application{
Expand Down

0 comments on commit 6cb545e

Please sign in to comment.