feat: Add max_file_size to Snowflake config (#5377)

nanohanno · hkuepers · web-flow · commit e8cdf5d12847 · 2025-05-21T06:52:50.000-04:00
Add max_file_size to Snowflake config

Signed-off-by: hkuepers &lt;hanno.kuepers@ratepay.com&gt;
Co-authored-by: hkuepers &lt;hanno.kuepers@ratepay.com&gt;
diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py
@@ -130,6 +130,9 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel):
 
     convert_timestamp_columns: Optional[bool] = None
     """ Convert timestamp columns on export to a Parquet-supported format """
+
+    max_file_size: Optional[int] = None
+    """ Upper size limit (in bytes) of each file that is offloaded. Default: 16777216"""
     model_config = ConfigDict(populate_by_name=True)
 
 
@@ -616,6 +619,9 @@ def to_remote_storage(self) -> List[str]:
               DETAILED_OUTPUT = TRUE
               HEADER = TRUE
         """
+        if (max_file_size := self.config.offline_store.max_file_size) is not None:
+            query += f"\nMAX_FILE_SIZE = {max_file_size}"
+
         cursor = execute_snowflake_statement(self.snowflake_conn, query)
         # s3gov schema is used by Snowflake in AWS govcloud regions
         # remove gov portion from schema and pass it to online store upload
