Abstract persistent files through Apache OpenDAL (#5626)

* Abstract file access through Apache OpenDAL

* Add AWS S3 support via OpenDAL for data files

* PR improvements

* Additional PR improvements

* Config setting comments for local/remote data locations

src/api/core/sends.rs

@@ -1,4 +1,5 @@
 use std::path::Path;
+use std::time::Duration;
 
 use chrono::{DateTime, TimeDelta, Utc};
 use num_traits::ToPrimitive;
@@ -12,8 +13,9 @@ use serde_json::Value;
 use crate::{
     api::{ApiResult, EmptyResult, JsonResult, Notify, UpdateType},
     auth::{ClientIp, Headers, Host},
+    config::PathType,
     db::{models::*, DbConn, DbPool},
-    util::NumberOrString,
+    util::{save_temp_file, NumberOrString},
     CONFIG,
 };
 
@@ -228,7 +230,7 @@ async fn post_send_file(data: Form<UploadData<'_>>, headers: Headers, mut conn:
 
     let UploadData {
         model,
-        mut data,
+        data,
     } = data.into_inner();
     let model = model.into_inner();
 
@@ -268,13 +270,8 @@ async fn post_send_file(data: Form<UploadData<'_>>, headers: Headers, mut conn:
     }
 
     let file_id = crate::crypto::generate_send_file_id();
-    let folder_path = tokio::fs::canonicalize(&CONFIG.sends_folder()).await?.join(&send.uuid);
-    let file_path = folder_path.join(&file_id);
-    tokio::fs::create_dir_all(&folder_path).await?;
 
-    if let Err(_err) = data.persist_to(&file_path).await {
-        data.move_copy_to(file_path).await?
-    }
+    save_temp_file(PathType::Sends, &format!("{}/{file_id}", send.uuid), data, true).await?;
 
     let mut data_value: Value = serde_json::from_str(&send.data)?;
     if let Some(o) = data_value.as_object_mut() {
@@ -381,7 +378,7 @@ async fn post_send_file_v2_data(
 ) -> EmptyResult {
     enforce_disable_send_policy(&headers, &mut conn).await?;
 
-    let mut data = data.into_inner();
+    let data = data.into_inner();
 
     let Some(send) = Send::find_by_uuid_and_user(&send_id, &headers.user.uuid, &mut conn).await else {
         err!("Send not found. Unable to save the file.", "Invalid send uuid or does not belong to user.")
@@ -424,19 +421,9 @@ async fn post_send_file_v2_data(
         err!("Send file size does not match.", format!("Expected a file size of {} got {size}", send_data.size));
     }
 
-    let folder_path = tokio::fs::canonicalize(&CONFIG.sends_folder()).await?.join(send_id);
-    let file_path = folder_path.join(file_id);
+    let file_path = format!("{send_id}/{file_id}");
 
-    // Check if the file already exists, if that is the case do not overwrite it
-    if tokio::fs::metadata(&file_path).await.is_ok() {
-        err!("Send file has already been uploaded.", format!("File {file_path:?} already exists"))
-    }
-
-    tokio::fs::create_dir_all(&folder_path).await?;
-
-    if let Err(_err) = data.data.persist_to(&file_path).await {
-        data.data.move_copy_to(file_path).await?
-    }
+    save_temp_file(PathType::Sends, &file_path, data.data, false).await?;
 
     nt.send_send_update(
         UpdateType::SyncSendCreate,
@@ -569,15 +556,26 @@ async fn post_access_file(
     )
     .await;
 
-    let token_claims = crate::auth::generate_send_claims(&send_id, &file_id);
-    let token = crate::auth::encode_jwt(&token_claims);
     Ok(Json(json!({
         "object": "send-fileDownload",
         "id": file_id,
-        "url": format!("{}/api/sends/{send_id}/{file_id}?t={token}", &host.host)
+        "url": download_url(&host, &send_id, &file_id).await?,
     })))
 }
 
+async fn download_url(host: &Host, send_id: &SendId, file_id: &SendFileId) -> Result<String, crate::Error> {
+    let operator = CONFIG.opendal_operator_for_path_type(PathType::Sends)?;
+
+    if operator.info().scheme() == opendal::Scheme::Fs {
+        let token_claims = crate::auth::generate_send_claims(send_id, file_id);
+        let token = crate::auth::encode_jwt(&token_claims);
+
+        Ok(format!("{}/api/sends/{send_id}/{file_id}?t={token}", &host.host))
+    } else {
+        Ok(operator.presign_read(&format!("{send_id}/{file_id}"), Duration::from_secs(5 * 60)).await?.uri().to_string())
+    }
+}
+
 #[get("/sends/<send_id>/<file_id>?<t>")]
 async fn download_send(send_id: SendId, file_id: SendFileId, t: &str) -> Option<NamedFile> {
     if let Ok(claims) = crate::auth::decode_send(t) {