refactor: merge parquet files interface

.github/workflows/unit-tests.yml

@@ -47,7 +47,7 @@ jobs:
         run: rustc --print cfg
 
       - name: Setup cargo config
-        run: cargo --config net.git-fetch-with-cli=true clippy -- -A clippy::question-mark -D warnings
+        run: cargo --config net.git-fetch-with-cli=true clippy -- -D warnings
 
       - name: Run unit tests
         run: ./coverage.sh

Cargo.toml

@@ -126,7 +126,7 @@ rayon.workspace = true
 regex.workspace = true
 regex-syntax.workspace = true
 reqwest.workspace = true
-rust-embed-for-web = "11.1"
+rust-embed-for-web = "11.2.1"
 segment.workspace = true
 serde.workspace = true
 serde_json.workspace = true

deploy/build/buildspec-test.yml

@@ -29,5 +29,5 @@ phases:
       - touch web/dist/index.html
 
       # run tests
-      - cargo --config net.git-fetch-with-cli=true clippy -- -A clippy::question-mark -D warnings
+      - cargo --config net.git-fetch-with-cli=true clippy -- -D warnings
       - ./coverage.sh

src/config/src/utils/parquet.rs

@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -111,6 +111,28 @@ pub fn new_parquet_writer<'a>(
     AsyncArrowWriter::try_new(buf, schema.clone(), Some(writer_props)).unwrap()
 }
 
+pub async fn write_recordbatch_to_parquet(
+    schema: Arc<Schema>,
+    record_batches: &[RecordBatch],
+    bloom_filter_fields: &[String],
+    full_text_search_fields: &[String],
+    metadata: &FileMeta,
+) -> Result<Vec<u8>, anyhow::Error> {
+    let mut buf = Vec::new();
+    let mut writer = new_parquet_writer(
+        &mut buf,
+        &schema,
+        bloom_filter_fields,
+        full_text_search_fields,
+        metadata,
+    );
+    for batch in record_batches {
+        writer.write(batch).await?;
+    }
+    writer.close().await?;
+    Ok(buf)
+}
+
 // parse file key to get stream_key, date_key, file_name
 pub fn parse_file_key_columns(key: &str) -> Result<(String, String, String), anyhow::Error> {
     // eg: files/default/logs/olympics/2022/10/03/10/6982652937134804993_1.parquet