Abstract persistent files through Apache OpenDAL (#5626)

* Abstract file access through Apache OpenDAL

* Add AWS S3 support via OpenDAL for data files

* PR improvements

* Additional PR improvements

* Config setting comments for local/remote data locations

Cargo.toml

@@ -32,6 +32,7 @@ enable_mimalloc = ["dep:mimalloc"]
 # You also need to set an env variable `QUERY_LOGGER=1` to fully activate this so you do not have to re-compile
 # if you want to turn off the logging for a specific run.
 query_logger = ["dep:diesel_logger"]
+s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:anyhow", "dep:reqsign"]
 
 # Enable unstable features, requires nightly
 # Currently only used to enable rusts official ip support
@@ -73,6 +74,7 @@ dashmap = "6.1.0"
 # Async futures
 futures = "0.3.31"
 tokio = { version = "1.45.1", features = ["rt-multi-thread", "fs", "io-util", "parking_lot", "time", "signal", "net"] }
+tokio-util = { version = "0.7.15", features = ["compat"]}
 
 # A generic serialization/deserialization framework
 serde = { version = "1.0.219", features = ["derive"] }
@@ -176,6 +178,15 @@ rpassword = "7.4.0"
 # Loading a dynamic CSS Stylesheet
 grass_compiler = { version = "0.13.4", default-features = false }
 
+# File are accessed through Apache OpenDAL
+opendal = { version = "0.53.2", features = ["services-fs"] }
+
+# For retrieving AWS credentials, including temporary SSO credentials
+anyhow = { version = "1.0.98", optional = true }
+aws-config = { version = "1.6.3", features = ["behavior-version-latest"], optional = true }
+aws-credential-types = { version = "1.2.3", optional = true }
+reqsign = { version = "0.16.3", optional = true }
+
 # Strip debuginfo from the release builds
 # The debug symbols are to provide better panic traces
 # Also enable fat LTO and use 1 codegen unit for optimizations