From 22aea48cc596a4b5e9ca78f131e660d033694204 Mon Sep 17 00:00:00 2001 From: trisua Date: Fri, 18 Jul 2025 00:14:52 -0400 Subject: [PATCH] add: better app data queries --- crates/app/src/macros.rs | 9 +- crates/app/src/public/html/developer/app.lisp | 2 +- .../app/src/public/html/developer/home.lisp | 5 +- .../app/src/public/html/developer/link.lisp | 7 ++ crates/app/src/routes/api/v1/app_data.rs | 86 +++++++++++++++---- crates/app/src/routes/api/v1/mod.rs | 4 +- crates/core/src/database/app_data.rs | 59 ++++++++++--- crates/core/src/database/apps.rs | 29 ++++--- .../database/drivers/sql/create_app_data.sql | 1 - .../src/database/drivers/sql/create_apps.sql | 3 +- crates/core/src/model/apps.rs | 28 ++++-- sql_changes/apps_data_used.sql | 2 +- 12 files changed, 175 insertions(+), 60 deletions(-) diff --git a/crates/app/src/macros.rs b/crates/app/src/macros.rs index 0edafbb..fd141ea 100644 --- a/crates/app/src/macros.rs +++ b/crates/app/src/macros.rs @@ -422,12 +422,9 @@ macro_rules! ignore_users_gen { #[macro_export] macro_rules! get_app_from_key { - ($db:ident, $jar:ident) => { - if let Some(token) = $jar.get("Atto-Secret-Key") { - match $db - .get_app_by_api_key(&token.to_string().replace("Atto-Secret-Key=", "")) - .await - { + ($db:ident, $headers:ident) => { + if let Some(token) = $headers.get("Atto-Secret-Key") { + match $db.get_app_by_api_key(token.to_str().unwrap()).await { Ok(x) => Some(x), Err(_) => None, } diff --git a/crates/app/src/public/html/developer/app.lisp b/crates/app/src/public/html/developer/app.lisp index d01e9de..6795001 100644 --- a/crates/app/src/public/html/developer/app.lisp +++ b/crates/app/src/public/html/developer/app.lisp @@ -19,7 +19,7 @@ (div ("class" "card flex flex-col gap-2") (p ("class" "fade") (text "App data keys are not included in this metric, only stored values count towards your limit.")) - (text "{% set percentage = (data_limit / app.data_used) * 100 %}") + (text "{% set percentage = (app.data_used / data_limit) * 100 %}") (div ("class" "progress_bar") (div ("class" "poll_bar") ("style" "width: {{ percentage }}%"))) (div ("class" "w-full flex justify-between items-center") diff --git a/crates/app/src/public/html/developer/home.lisp b/crates/app/src/public/html/developer/home.lisp index fb00c7e..d96be6f 100644 --- a/crates/app/src/public/html/developer/home.lisp +++ b/crates/app/src/public/html/developer/home.lisp @@ -47,13 +47,12 @@ ("class" "flex flex-col gap-1") (label ("for" "title") - (text "{{ text \"developer:label.redirect\" }}")) + (text "{{ text \"developer:label.redirect\" }} (optional)")) (input ("type" "url") ("name" "redirect") ("id" "redirect") ("placeholder" "redirect URL") - ("required" "") ("minlength" "2") ("maxlength" "32"))) (button @@ -125,7 +124,7 @@ body: JSON.stringify({ title: e.target.title.value, homepage: e.target.homepage.value, - redirect: e.target.redirect.value, + redirect: e.target.redirect.value || \"\", }), }) .then((res) => res.json()) diff --git a/crates/app/src/public/html/developer/link.lisp b/crates/app/src/public/html/developer/link.lisp index 5d46c87..2c94309 100644 --- a/crates/app/src/public/html/developer/link.lisp +++ b/crates/app/src/public/html/developer/link.lisp @@ -39,6 +39,13 @@ (str (text "dialog:action.cancel"))))))) (script (text "setTimeout(() => { + // {% if app.redirect|length == 0 %} + alert(\"App has an invalid redirect. Please contact the owner for help.\"); + window.close(); + return; + // {% endif %} + + // ... globalThis.authorize = async (event) => { if ( !(await trigger(\"atto::confirm\", [ diff --git a/crates/app/src/routes/api/v1/app_data.rs b/crates/app/src/routes/api/v1/app_data.rs index b4c0d03..c2983f1 100644 --- a/crates/app/src/routes/api/v1/app_data.rs +++ b/crates/app/src/routes/api/v1/app_data.rs @@ -3,20 +3,19 @@ use crate::{ routes::api::v1::{InsertAppData, QueryAppData, UpdateAppDataValue}, State, }; -use axum::{Extension, Json, extract::Path, response::IntoResponse}; -use axum_extra::extract::CookieJar; +use axum::{extract::Path, http::HeaderMap, response::IntoResponse, Extension, Json}; use tetratto_core::model::{ - apps::{AppData, AppDataQuery}, + apps::{AppData, AppDataQuery, AppDataQueryResult}, ApiReturn, Error, }; pub async fn query_request( - jar: CookieJar, + headers: HeaderMap, Extension(data): Extension, Json(req): Json, ) -> impl IntoResponse { let data = &(data.read().await).0; - let app = match get_app_from_key!(data, jar) { + let app = match get_app_from_key!(data, headers) { Some(x) => x, None => return Json(Error::NotAllowed.into()), }; @@ -39,12 +38,12 @@ pub async fn query_request( } pub async fn create_request( - jar: CookieJar, + headers: HeaderMap, Extension(data): Extension, Json(req): Json, ) -> impl IntoResponse { let data = &(data.read().await).0; - let app = match get_app_from_key!(data, jar) { + let app = match get_app_from_key!(data, headers) { Some(x) => x, None => return Json(Error::NotAllowed.into()), }; @@ -61,7 +60,7 @@ pub async fn create_request( } // ... - if let Err(e) = data.update_app_data_used(app.id, new_size as i32).await { + if let Err(e) = data.add_app_data_used(app.id, req.value.len() as i32).await { return Json(e.into()); } @@ -71,7 +70,7 @@ pub async fn create_request( { Ok(s) => Json(ApiReturn { ok: true, - message: "App created".to_string(), + message: "Data inserted".to_string(), payload: s.id.to_string(), }), Err(e) => Json(e.into()), @@ -79,13 +78,13 @@ pub async fn create_request( } pub async fn update_value_request( - jar: CookieJar, + headers: HeaderMap, Extension(data): Extension, Path(id): Path, Json(req): Json, ) -> impl IntoResponse { let data = &(data.read().await).0; - let app = match get_app_from_key!(data, jar) { + let app = match get_app_from_key!(data, headers) { Some(x) => x, None => return Json(Error::NotAllowed.into()), }; @@ -109,7 +108,7 @@ pub async fn update_value_request( } // ... - if let Err(e) = data.update_app_data_used(app.id, new_size as i32).await { + if let Err(e) = data.add_app_data_used(app.id, req.value.len() as i32).await { return Json(e.into()); } @@ -124,12 +123,12 @@ pub async fn update_value_request( } pub async fn delete_request( - jar: CookieJar, + headers: HeaderMap, Extension(data): Extension, Path(id): Path, ) -> impl IntoResponse { let data = &(data.read().await).0; - let app = match get_app_from_key!(data, jar) { + let app = match get_app_from_key!(data, headers) { Some(x) => x, None => return Json(Error::NotAllowed.into()), }; @@ -141,7 +140,7 @@ pub async fn delete_request( // ... if let Err(e) = data - .update_app_data_used(app.id, (app.data_used - app_data.value.len()) as i32) + .add_app_data_used(app.id, -(app_data.value.len() as i32)) .await { return Json(e.into()); @@ -156,3 +155,60 @@ pub async fn delete_request( Err(e) => Json(e.into()), } } + +pub async fn delete_query_request( + headers: HeaderMap, + Extension(data): Extension, + Json(req): Json, +) -> impl IntoResponse { + let data = &(data.read().await).0; + let app = match get_app_from_key!(data, headers) { + Some(x) => x, + None => return Json(Error::NotAllowed.into()), + }; + + // ... + let rows = match data + .query_app_data(AppDataQuery { + app: app.id, + query: req.query.clone(), + mode: req.mode.clone(), + }) + .await + { + Ok(x) => match x { + AppDataQueryResult::One(x) => vec![x], + AppDataQueryResult::Many(x) => x, + }, + Err(e) => return Json(e.into()), + }; + + let mut subtract_amount: usize = 0; + for row in &rows { + subtract_amount += row.value.len(); + } + drop(rows); + + if let Err(e) = data + .add_app_data_used(app.id, -(subtract_amount as i32)) + .await + { + return Json(e.into()); + } + + match data + .query_delete_app_data(AppDataQuery { + app: app.id, + query: req.query, + mode: req.mode, + }) + .await + { + Ok(_) => Json(ApiReturn { + ok: true, + message: "Data deleted".to_string(), + payload: (), + }), + Err(e) => Json(e.into()), + } +} diff --git a/crates/app/src/routes/api/v1/mod.rs b/crates/app/src/routes/api/v1/mod.rs index 8b276dd..9e48f8d 100644 --- a/crates/app/src/routes/api/v1/mod.rs +++ b/crates/app/src/routes/api/v1/mod.rs @@ -433,9 +433,10 @@ pub fn routes() -> Router { .route("/apps/{id}/roll", post(apps::roll_api_key_request)) // app data .route("/app_data", post(app_data::create_request)) - .route("/app_data/query", post(app_data::query_request)) .route("/app_data/{id}", delete(app_data::delete_request)) .route("/app_data/{id}/value", post(app_data::update_value_request)) + .route("/app_data/query", post(app_data::query_request)) + .route("/app_data/query", delete(app_data::delete_query_request)) // warnings .route("/warnings/{id}", get(auth::user_warnings::get_request)) .route("/warnings/{id}", post(auth::user_warnings::create_request)) @@ -987,6 +988,7 @@ pub struct UpdatePostIsOpen { pub struct CreateApp { pub title: String, pub homepage: String, + #[serde(default)] pub redirect: String, } diff --git a/crates/core/src/database/app_data.rs b/crates/core/src/database/app_data.rs index 6ea4f63..7ddece5 100644 --- a/crates/core/src/database/app_data.rs +++ b/crates/core/src/database/app_data.rs @@ -12,9 +12,9 @@ impl DataManager { pub(crate) fn get_app_data_from_row(x: &PostgresRow) -> AppData { AppData { id: get!(x->0(i64)) as usize, - app: get!(x->2(i64)) as usize, - key: get!(x->3(String)), - value: get!(x->4(String)), + app: get!(x->1(i64)) as usize, + key: get!(x->2(String)), + value: get!(x->3(String)), } } @@ -44,10 +44,7 @@ impl DataManager { Ok(res.unwrap()) } - /// Get all app_data by owner. - /// - /// # Arguments - /// * `id` - the ID of the user to fetch app_data for + /// Get all app_data by the given query. pub async fn query_app_data(&self, query: AppDataQuery) -> Result { let conn = match self.0.connect().await { Ok(c) => c, @@ -57,12 +54,13 @@ impl DataManager { let query_str = query.to_string().replace( "%q%", &match query.query { - AppDataSelectQuery::Like(_, _) => format!("v LIKE $1"), + AppDataSelectQuery::KeyIs(_) => format!("k = $1"), + AppDataSelectQuery::LikeJson(_, _) => format!("v LIKE $1"), }, ); let res = match query.mode { - AppDataSelectMode::One => AppDataQueryResult::One( + AppDataSelectMode::One(_) => AppDataQueryResult::One( match query_row!(&conn, &query_str, params![&query.query.to_string()], |x| { Ok(Self::get_app_data_from_row(x)) }) { @@ -70,7 +68,15 @@ impl DataManager { Err(_) => return Err(Error::GeneralNotFound("app_data".to_string())), }, ), - AppDataSelectMode::Many(_, _, _) => AppDataQueryResult::Many( + AppDataSelectMode::Many(_, _) => AppDataQueryResult::Many( + match query_rows!(&conn, &query_str, params![&query.query.to_string()], |x| { + Self::get_app_data_from_row(x) + }) { + Ok(x) => x, + Err(_) => return Err(Error::GeneralNotFound("app_data".to_string())), + }, + ), + AppDataSelectMode::ManyJson(_, _, _) => AppDataQueryResult::Many( match query_rows!(&conn, &query_str, params![&query.query.to_string()], |x| { Self::get_app_data_from_row(x) }) { @@ -83,6 +89,35 @@ impl DataManager { Ok(res) } + /// Delete all app_data matched by the given query. + pub async fn query_delete_app_data(&self, query: AppDataQuery) -> Result<()> { + let conn = match self.0.connect().await { + Ok(c) => c, + Err(e) => return Err(Error::DatabaseConnection(e.to_string())), + }; + + let query_str = query + .to_string() + .replace( + "%q%", + &match query.query { + AppDataSelectQuery::KeyIs(_) => format!("k = $1"), + AppDataSelectQuery::LikeJson(_, _) => format!("v LIKE $1"), + }, + ) + .replace("SELECT * FROM", "SELECT id FROM"); + + if let Err(e) = execute!( + &conn, + &format!("DELETE FROM app_data WHERE id IN ({query_str})"), + params![&query.query.to_string()] + ) { + return Err(Error::MiscError(e.to_string())); + } + + Ok(()) + } + const MAXIMUM_FREE_APP_DATA: usize = 5; const MAXIMUM_DATA_SIZE: usize = 205_000; @@ -101,9 +136,9 @@ impl DataManager { } if data.value.len() < 2 { - return Err(Error::DataTooShort("key".to_string())); + return Err(Error::DataTooShort("value".to_string())); } else if data.value.len() > Self::MAXIMUM_DATA_SIZE { - return Err(Error::DataTooLong("key".to_string())); + return Err(Error::DataTooLong("value".to_string())); } // check number of app_data diff --git a/crates/core/src/database/apps.rs b/crates/core/src/database/apps.rs index 4915907..1fa5f31 100644 --- a/crates/core/src/database/apps.rs +++ b/crates/core/src/database/apps.rs @@ -29,7 +29,7 @@ impl DataManager { } auto_method!(get_app_by_id(usize as i64)@get_app_from_row -> "SELECT * FROM apps WHERE id = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app:{}"); - auto_method!(get_app_by_api_key(&str)@get_app_from_row -> "SELECT * FROM apps WHERE api_key = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app:{}"); + auto_method!(get_app_by_api_key(&str)@get_app_from_row -> "SELECT * FROM apps WHERE api_key = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app_k:{}"); /// Get all apps by user. /// @@ -134,7 +134,7 @@ impl DataManager { return Err(Error::DatabaseError(e.to_string())); } - self.0.1.remove(format!("atto.app:{}", id)).await; + self.cache_clear_app(&app).await; // remove data let res = execute!( @@ -151,14 +151,21 @@ impl DataManager { Ok(()) } - auto_method!(update_app_title(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET title = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_homepage(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET homepage = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_redirect(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET redirect = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_quota_status(AppQuota) -> "UPDATE apps SET quota_status = $1 WHERE id = $2" --serde --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_scopes(Vec)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET scopes = $1 WHERE id = $2" --serde --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_api_key(&str) -> "UPDATE apps SET api_key = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}"); - auto_method!(update_app_data_used(i32) -> "UPDATE apps SET data_used = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}"); + pub async fn cache_clear_app(&self, app: &ThirdPartyApp) { + self.0.1.remove(format!("atto.app:{}", app.id)).await; + self.0.1.remove(format!("atto.app_k:{}", app.api_key)).await; + } - auto_method!(incr_app_grants() -> "UPDATE apps SET grants = grants + 1 WHERE id = $1" --cache-key-tmpl="atto.app:{}" --incr); - auto_method!(decr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants - 1 WHERE id = $1" --cache-key-tmpl="atto.app:{}" --decr=grants); + auto_method!(update_app_title(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET title = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + auto_method!(update_app_homepage(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET homepage = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + auto_method!(update_app_redirect(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET redirect = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + auto_method!(update_app_quota_status(AppQuota)@get_app_by_id -> "UPDATE apps SET quota_status = $1 WHERE id = $2" --serde --cache-key-tmpl=cache_clear_app); + auto_method!(update_app_scopes(Vec)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET scopes = $1 WHERE id = $2" --serde --cache-key-tmpl=cache_clear_app); + auto_method!(update_app_api_key(&str)@get_app_by_id -> "UPDATE apps SET api_key = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + + auto_method!(update_app_data_used(i32)@get_app_by_id -> "UPDATE apps SET data_used = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + auto_method!(add_app_data_used(i32)@get_app_by_id -> "UPDATE apps SET data_used = data_used + $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app); + + auto_method!(incr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants + 1 WHERE id = $1" --cache-key-tmpl=cache_clear_app --incr); + auto_method!(decr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants - 1 WHERE id = $1" --cache-key-tmpl=cache_clear_app --decr=grants); } diff --git a/crates/core/src/database/drivers/sql/create_app_data.sql b/crates/core/src/database/drivers/sql/create_app_data.sql index 28a8379..64cdd3f 100644 --- a/crates/core/src/database/drivers/sql/create_app_data.sql +++ b/crates/core/src/database/drivers/sql/create_app_data.sql @@ -1,6 +1,5 @@ CREATE TABLE IF NOT EXISTS app_data ( id BIGINT NOT NULL PRIMARY KEY, - owner BIGINT NOT NULL, app BIGINT NOT NULL, k TEXT NOT NULL, v TEXT NOT NULL diff --git a/crates/core/src/database/drivers/sql/create_apps.sql b/crates/core/src/database/drivers/sql/create_apps.sql index 575ce5c..d01ed41 100644 --- a/crates/core/src/database/drivers/sql/create_apps.sql +++ b/crates/core/src/database/drivers/sql/create_apps.sql @@ -8,5 +8,6 @@ CREATE TABLE IF NOT EXISTS apps ( quota_status TEXT NOT NULL, banned INT NOT NULL, grants INT NOT NULL, - scopes TEXT NOT NULL + scopes TEXT NOT NULL, + data_used INT NOT NULL CHECK (data_used >= 0) ) diff --git a/crates/core/src/model/apps.rs b/crates/core/src/model/apps.rs index b48b0ed..bca5c81 100644 --- a/crates/core/src/model/apps.rs +++ b/crates/core/src/model/apps.rs @@ -146,34 +146,46 @@ impl AppData { #[derive(Serialize, Deserialize, Debug, Clone)] pub enum AppDataSelectQuery { - Like(String, String), + KeyIs(String), + LikeJson(String, String), } impl Display for AppDataSelectQuery { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&match self { - Self::Like(k, v) => format!("%\"{k}\":\"{v}\"%"), + Self::KeyIs(k) => k.to_owned(), + Self::LikeJson(k, v) => format!("%\"{k}\":\"{v}\"%"), }) } } #[derive(Serialize, Deserialize, Debug, Clone)] pub enum AppDataSelectMode { - /// Select a single row. - One, + /// Select a single row (with offset). + One(usize), + /// Select multiple rows at once. + /// + /// `(limit, offset)` + Many(usize, usize), /// Select multiple rows at once. /// /// `(order by top level key, limit, offset)` - Many(String, usize, usize), + ManyJson(String, usize, usize), } impl Display for AppDataSelectMode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&match self { - Self::One => "LIMIT 1".to_string(), - Self::Many(order_by_top_level_key, limit, offset) => { + Self::One(offset) => format!("LIMIT 1 OFFSET {offset}"), + Self::Many(limit, offset) => { format!( - "ORDER BY v::jsonb->>'{order_by_top_level_key}' LIMIT {} OFFSET {offset}", + "LIMIT {} OFFSET {offset}", + if *limit > 1024 { 1024 } else { *limit } + ) + } + Self::ManyJson(order_by_top_level_key, limit, offset) => { + format!( + "ORDER BY v::jsonb->>'{order_by_top_level_key}' DESC LIMIT {} OFFSET {offset}", if *limit > 1024 { 1024 } else { *limit } ) } diff --git a/sql_changes/apps_data_used.sql b/sql_changes/apps_data_used.sql index 77202a0..a86da50 100644 --- a/sql_changes/apps_data_used.sql +++ b/sql_changes/apps_data_used.sql @@ -1,2 +1,2 @@ ALTER TABLE apps -ADD COLUMN data_used INT NOT NULL DEFAULT 0; +ADD COLUMN data_used INT NOT NULL DEFAULT 0 CHECK (data_used >= 0);