add: better app data queries

crates/app/src/macros.rs

@@ -422,12 +422,9 @@ macro_rules! ignore_users_gen {
 
 #[macro_export]
 macro_rules! get_app_from_key {
-    ($db:ident, $jar:ident) => {
+    ($db:ident, $headers:ident) => {
-        if let Some(token) = $jar.get("Atto-Secret-Key") {
+        if let Some(token) = $headers.get("Atto-Secret-Key") {
-            match $db
+            match $db.get_app_by_api_key(token.to_str().unwrap()).await {
-                .get_app_by_api_key(&token.to_string().replace("Atto-Secret-Key=", ""))
-                .await
-            {
                 Ok(x) => Some(x),
                 Err(_) => None,
             }

crates/app/src/public/html/developer/app.lisp

@@ -19,7 +19,7 @@
                 (div
                     ("class" "card flex flex-col gap-2")
                     (p ("class" "fade") (text "App data keys are not included in this metric, only stored values count towards your limit."))
-                    (text "{% set percentage = (data_limit / app.data_used) * 100 %}")
+                    (text "{% set percentage = (app.data_used / data_limit) * 100 %}")
                     (div ("class" "progress_bar") (div ("class" "poll_bar") ("style" "width: {{ percentage }}%")))
                     (div
                         ("class" "w-full flex justify-between items-center")

crates/app/src/public/html/developer/home.lisp

@@ -47,13 +47,12 @@
                 ("class" "flex flex-col gap-1")
                 (label
                     ("for" "title")
-                    (text "{{ text \"developer:label.redirect\" }}"))
+                    (text "{{ text \"developer:label.redirect\" }} (optional)"))
                 (input
                     ("type" "url")
                     ("name" "redirect")
                     ("id" "redirect")
                     ("placeholder" "redirect URL")
-                    ("required" "")
                     ("minlength" "2")
                     ("maxlength" "32")))
             (button
@@ -125,7 +124,7 @@
             body: JSON.stringify({
                 title: e.target.title.value,
                 homepage: e.target.homepage.value,
-                redirect: e.target.redirect.value,
+                redirect: e.target.redirect.value || \"\",
             }),
         })
             .then((res) => res.json())

crates/app/src/public/html/developer/link.lisp

@@ -39,6 +39,13 @@
                     (str (text "dialog:action.cancel")))))))
 (script
     (text "setTimeout(() => {
+        // {% if app.redirect|length == 0 %}
+        alert(\"App has an invalid redirect. Please contact the owner for help.\");
+        window.close();
+        return;
+        // {% endif %}
+
+        // ...
         globalThis.authorize = async (event) => {
             if (
                 !(await trigger(\"atto::confirm\", [

crates/app/src/routes/api/v1/app_data.rs

@@ -3,20 +3,19 @@ use crate::{
     routes::api::v1::{InsertAppData, QueryAppData, UpdateAppDataValue},
     State,
 };
-use axum::{Extension, Json, extract::Path, response::IntoResponse};
+use axum::{extract::Path, http::HeaderMap, response::IntoResponse, Extension, Json};
-use axum_extra::extract::CookieJar;
 use tetratto_core::model::{
-    apps::{AppData, AppDataQuery},
+    apps::{AppData, AppDataQuery, AppDataQueryResult},
     ApiReturn, Error,
 };
 
 pub async fn query_request(
-    jar: CookieJar,
+    headers: HeaderMap,
     Extension(data): Extension<State>,
     Json(req): Json<QueryAppData>,
 ) -> impl IntoResponse {
     let data = &(data.read().await).0;
-    let app = match get_app_from_key!(data, jar) {
+    let app = match get_app_from_key!(data, headers) {
         Some(x) => x,
         None => return Json(Error::NotAllowed.into()),
     };
@@ -39,12 +38,12 @@ pub async fn query_request(
 }
 
 pub async fn create_request(
-    jar: CookieJar,
+    headers: HeaderMap,
     Extension(data): Extension<State>,
     Json(req): Json<InsertAppData>,
 ) -> impl IntoResponse {
     let data = &(data.read().await).0;
-    let app = match get_app_from_key!(data, jar) {
+    let app = match get_app_from_key!(data, headers) {
         Some(x) => x,
         None => return Json(Error::NotAllowed.into()),
     };
@@ -61,7 +60,7 @@ pub async fn create_request(
     }
 
     // ...
-    if let Err(e) = data.update_app_data_used(app.id, new_size as i32).await {
+    if let Err(e) = data.add_app_data_used(app.id, req.value.len() as i32).await {
         return Json(e.into());
     }
 
@@ -71,7 +70,7 @@ pub async fn create_request(
     {
         Ok(s) => Json(ApiReturn {
             ok: true,
-            message: "App created".to_string(),
+            message: "Data inserted".to_string(),
             payload: s.id.to_string(),
         }),
         Err(e) => Json(e.into()),
@@ -79,13 +78,13 @@ pub async fn create_request(
 }
 
 pub async fn update_value_request(
-    jar: CookieJar,
+    headers: HeaderMap,
     Extension(data): Extension<State>,
     Path(id): Path<usize>,
     Json(req): Json<UpdateAppDataValue>,
 ) -> impl IntoResponse {
     let data = &(data.read().await).0;
-    let app = match get_app_from_key!(data, jar) {
+    let app = match get_app_from_key!(data, headers) {
         Some(x) => x,
         None => return Json(Error::NotAllowed.into()),
     };
@@ -109,7 +108,7 @@ pub async fn update_value_request(
     }
 
     // ...
-    if let Err(e) = data.update_app_data_used(app.id, new_size as i32).await {
+    if let Err(e) = data.add_app_data_used(app.id, req.value.len() as i32).await {
         return Json(e.into());
     }
 
@@ -124,12 +123,12 @@ pub async fn update_value_request(
 }
 
 pub async fn delete_request(
-    jar: CookieJar,
+    headers: HeaderMap,
     Extension(data): Extension<State>,
     Path(id): Path<usize>,
 ) -> impl IntoResponse {
     let data = &(data.read().await).0;
-    let app = match get_app_from_key!(data, jar) {
+    let app = match get_app_from_key!(data, headers) {
         Some(x) => x,
         None => return Json(Error::NotAllowed.into()),
     };
@@ -141,7 +140,7 @@ pub async fn delete_request(
 
     // ...
     if let Err(e) = data
-        .update_app_data_used(app.id, (app.data_used - app_data.value.len()) as i32)
+        .add_app_data_used(app.id, -(app_data.value.len() as i32))
         .await
     {
         return Json(e.into());
@@ -156,3 +155,60 @@ pub async fn delete_request(
         Err(e) => Json(e.into()),
     }
 }
+
+pub async fn delete_query_request(
+    headers: HeaderMap,
+    Extension(data): Extension<State>,
+    Json(req): Json<QueryAppData>,
+) -> impl IntoResponse {
+    let data = &(data.read().await).0;
+    let app = match get_app_from_key!(data, headers) {
+        Some(x) => x,
+        None => return Json(Error::NotAllowed.into()),
+    };
+
+    // ...
+    let rows = match data
+        .query_app_data(AppDataQuery {
+            app: app.id,
+            query: req.query.clone(),
+            mode: req.mode.clone(),
+        })
+        .await
+    {
+        Ok(x) => match x {
+            AppDataQueryResult::One(x) => vec![x],
+            AppDataQueryResult::Many(x) => x,
+        },
+        Err(e) => return Json(e.into()),
+    };
+
+    let mut subtract_amount: usize = 0;
+    for row in &rows {
+        subtract_amount += row.value.len();
+    }
+    drop(rows);
+
+    if let Err(e) = data
+        .add_app_data_used(app.id, -(subtract_amount as i32))
+        .await
+    {
+        return Json(e.into());
+    }
+
+    match data
+        .query_delete_app_data(AppDataQuery {
+            app: app.id,
+            query: req.query,
+            mode: req.mode,
+        })
+        .await
+    {
+        Ok(_) => Json(ApiReturn {
+            ok: true,
+            message: "Data deleted".to_string(),
+            payload: (),
+        }),
+        Err(e) => Json(e.into()),
+    }
+}

crates/app/src/routes/api/v1/mod.rs

@@ -433,9 +433,10 @@ pub fn routes() -> Router {
         .route("/apps/{id}/roll", post(apps::roll_api_key_request))
         // app data
         .route("/app_data", post(app_data::create_request))
-        .route("/app_data/query", post(app_data::query_request))
         .route("/app_data/{id}", delete(app_data::delete_request))
         .route("/app_data/{id}/value", post(app_data::update_value_request))
+        .route("/app_data/query", post(app_data::query_request))
+        .route("/app_data/query", delete(app_data::delete_query_request))
         // warnings
         .route("/warnings/{id}", get(auth::user_warnings::get_request))
         .route("/warnings/{id}", post(auth::user_warnings::create_request))
@@ -987,6 +988,7 @@ pub struct UpdatePostIsOpen {
 pub struct CreateApp {
     pub title: String,
     pub homepage: String,
+    #[serde(default)]
     pub redirect: String,
 }
 

crates/core/src/database/app_data.rs

@@ -12,9 +12,9 @@ impl DataManager {
     pub(crate) fn get_app_data_from_row(x: &PostgresRow) -> AppData {
         AppData {
             id: get!(x->0(i64)) as usize,
-            app: get!(x->2(i64)) as usize,
+            app: get!(x->1(i64)) as usize,
-            key: get!(x->3(String)),
+            key: get!(x->2(String)),
-            value: get!(x->4(String)),
+            value: get!(x->3(String)),
         }
     }
 
@@ -44,10 +44,7 @@ impl DataManager {
         Ok(res.unwrap())
     }
 
-    /// Get all app_data by owner.
+    /// Get all app_data by the given query.
-    ///
-    /// # Arguments
-    /// * `id` - the ID of the user to fetch app_data for
     pub async fn query_app_data(&self, query: AppDataQuery) -> Result<AppDataQueryResult> {
         let conn = match self.0.connect().await {
             Ok(c) => c,
@@ -57,12 +54,13 @@ impl DataManager {
         let query_str = query.to_string().replace(
             "%q%",
             &match query.query {
-                AppDataSelectQuery::Like(_, _) => format!("v LIKE $1"),
+                AppDataSelectQuery::KeyIs(_) => format!("k = $1"),
+                AppDataSelectQuery::LikeJson(_, _) => format!("v LIKE $1"),
             },
         );
 
         let res = match query.mode {
-            AppDataSelectMode::One => AppDataQueryResult::One(
+            AppDataSelectMode::One(_) => AppDataQueryResult::One(
                 match query_row!(&conn, &query_str, params![&query.query.to_string()], |x| {
                     Ok(Self::get_app_data_from_row(x))
                 }) {
@@ -70,7 +68,15 @@ impl DataManager {
                     Err(_) => return Err(Error::GeneralNotFound("app_data".to_string())),
                 },
             ),
-            AppDataSelectMode::Many(_, _, _) => AppDataQueryResult::Many(
+            AppDataSelectMode::Many(_, _) => AppDataQueryResult::Many(
+                match query_rows!(&conn, &query_str, params![&query.query.to_string()], |x| {
+                    Self::get_app_data_from_row(x)
+                }) {
+                    Ok(x) => x,
+                    Err(_) => return Err(Error::GeneralNotFound("app_data".to_string())),
+                },
+            ),
+            AppDataSelectMode::ManyJson(_, _, _) => AppDataQueryResult::Many(
                 match query_rows!(&conn, &query_str, params![&query.query.to_string()], |x| {
                     Self::get_app_data_from_row(x)
                 }) {
@@ -83,6 +89,35 @@ impl DataManager {
         Ok(res)
     }
 
+    /// Delete all app_data matched by the given query.
+    pub async fn query_delete_app_data(&self, query: AppDataQuery) -> Result<()> {
+        let conn = match self.0.connect().await {
+            Ok(c) => c,
+            Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
+        };
+
+        let query_str = query
+            .to_string()
+            .replace(
+                "%q%",
+                &match query.query {
+                    AppDataSelectQuery::KeyIs(_) => format!("k = $1"),
+                    AppDataSelectQuery::LikeJson(_, _) => format!("v LIKE $1"),
+                },
+            )
+            .replace("SELECT * FROM", "SELECT id FROM");
+
+        if let Err(e) = execute!(
+            &conn,
+            &format!("DELETE FROM app_data WHERE id IN ({query_str})"),
+            params![&query.query.to_string()]
+        ) {
+            return Err(Error::MiscError(e.to_string()));
+        }
+
+        Ok(())
+    }
+
     const MAXIMUM_FREE_APP_DATA: usize = 5;
     const MAXIMUM_DATA_SIZE: usize = 205_000;
 
@@ -101,9 +136,9 @@ impl DataManager {
         }
 
         if data.value.len() < 2 {
-            return Err(Error::DataTooShort("key".to_string()));
+            return Err(Error::DataTooShort("value".to_string()));
         } else if data.value.len() > Self::MAXIMUM_DATA_SIZE {
-            return Err(Error::DataTooLong("key".to_string()));
+            return Err(Error::DataTooLong("value".to_string()));
         }
 
         // check number of app_data

crates/core/src/database/apps.rs

@@ -29,7 +29,7 @@ impl DataManager {
     }
 
     auto_method!(get_app_by_id(usize as i64)@get_app_from_row -> "SELECT * FROM apps WHERE id = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app:{}");
-    auto_method!(get_app_by_api_key(&str)@get_app_from_row -> "SELECT * FROM apps WHERE api_key = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app:{}");
+    auto_method!(get_app_by_api_key(&str)@get_app_from_row -> "SELECT * FROM apps WHERE api_key = $1" --name="app" --returns=ThirdPartyApp --cache-key-tmpl="atto.app_k:{}");
 
     /// Get all apps by user.
     ///
@@ -134,7 +134,7 @@ impl DataManager {
             return Err(Error::DatabaseError(e.to_string()));
         }
 
-        self.0.1.remove(format!("atto.app:{}", id)).await;
+        self.cache_clear_app(&app).await;
 
         // remove data
         let res = execute!(
@@ -151,14 +151,21 @@ impl DataManager {
         Ok(())
     }
 
-    auto_method!(update_app_title(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET title = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}");
+    pub async fn cache_clear_app(&self, app: &ThirdPartyApp) {
-    auto_method!(update_app_homepage(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET homepage = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}");
+        self.0.1.remove(format!("atto.app:{}", app.id)).await;
-    auto_method!(update_app_redirect(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET redirect = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}");
+        self.0.1.remove(format!("atto.app_k:{}", app.api_key)).await;
-    auto_method!(update_app_quota_status(AppQuota) -> "UPDATE apps SET quota_status = $1 WHERE id = $2" --serde --cache-key-tmpl="atto.app:{}");
+    }
-    auto_method!(update_app_scopes(Vec<AppScope>)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET scopes = $1 WHERE id = $2" --serde --cache-key-tmpl="atto.app:{}");
+
-    auto_method!(update_app_api_key(&str) -> "UPDATE apps SET api_key = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}");
+    auto_method!(update_app_title(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET title = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
-    auto_method!(update_app_data_used(i32) -> "UPDATE apps SET data_used = $1 WHERE id = $2" --cache-key-tmpl="atto.app:{}");
+    auto_method!(update_app_homepage(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET homepage = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
-
+    auto_method!(update_app_redirect(&str)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET redirect = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
-    auto_method!(incr_app_grants() -> "UPDATE apps SET grants = grants + 1 WHERE id = $1" --cache-key-tmpl="atto.app:{}" --incr);
+    auto_method!(update_app_quota_status(AppQuota)@get_app_by_id -> "UPDATE apps SET quota_status = $1 WHERE id = $2" --serde --cache-key-tmpl=cache_clear_app);
-    auto_method!(decr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants - 1 WHERE id = $1" --cache-key-tmpl="atto.app:{}" --decr=grants);
+    auto_method!(update_app_scopes(Vec<AppScope>)@get_app_by_id:FinePermission::MANAGE_APPS; -> "UPDATE apps SET scopes = $1 WHERE id = $2" --serde --cache-key-tmpl=cache_clear_app);
+    auto_method!(update_app_api_key(&str)@get_app_by_id -> "UPDATE apps SET api_key = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
+
+    auto_method!(update_app_data_used(i32)@get_app_by_id -> "UPDATE apps SET data_used = $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
+    auto_method!(add_app_data_used(i32)@get_app_by_id -> "UPDATE apps SET data_used = data_used + $1 WHERE id = $2" --cache-key-tmpl=cache_clear_app);
+
+    auto_method!(incr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants + 1 WHERE id = $1" --cache-key-tmpl=cache_clear_app --incr);
+    auto_method!(decr_app_grants()@get_app_by_id -> "UPDATE apps SET grants = grants - 1 WHERE id = $1" --cache-key-tmpl=cache_clear_app --decr=grants);
 }

crates/core/src/database/drivers/sql/create_app_data.sql

@@ -1,6 +1,5 @@
 CREATE TABLE IF NOT EXISTS app_data (
     id BIGINT NOT NULL PRIMARY KEY,
-    owner BIGINT NOT NULL,
     app BIGINT NOT NULL,
     k TEXT NOT NULL,
     v TEXT NOT NULL

crates/core/src/database/drivers/sql/create_apps.sql

@@ -8,5 +8,6 @@ CREATE TABLE IF NOT EXISTS apps (
     quota_status TEXT NOT NULL,
     banned INT NOT NULL,
     grants INT NOT NULL,
-    scopes TEXT NOT NULL
+    scopes TEXT NOT NULL,
+    data_used INT NOT NULL CHECK (data_used >= 0)
 )

crates/core/src/model/apps.rs

@@ -146,34 +146,46 @@ impl AppData {
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum AppDataSelectQuery {
-    Like(String, String),
+    KeyIs(String),
+    LikeJson(String, String),
 }
 
 impl Display for AppDataSelectQuery {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str(&match self {
-            Self::Like(k, v) => format!("%\"{k}\":\"{v}\"%"),
+            Self::KeyIs(k) => k.to_owned(),
+            Self::LikeJson(k, v) => format!("%\"{k}\":\"{v}\"%"),
         })
     }
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum AppDataSelectMode {
-    /// Select a single row.
+    /// Select a single row (with offset).
-    One,
+    One(usize),
+    /// Select multiple rows at once.
+    ///
+    /// `(limit, offset)`
+    Many(usize, usize),
     /// Select multiple rows at once.
     ///
     /// `(order by top level key, limit, offset)`
-    Many(String, usize, usize),
+    ManyJson(String, usize, usize),
 }
 
 impl Display for AppDataSelectMode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_str(&match self {
-            Self::One => "LIMIT 1".to_string(),
+            Self::One(offset) => format!("LIMIT 1 OFFSET {offset}"),
-            Self::Many(order_by_top_level_key, limit, offset) => {
+            Self::Many(limit, offset) => {
                 format!(
-                    "ORDER BY v::jsonb->>'{order_by_top_level_key}' LIMIT {} OFFSET {offset}",
+                    "LIMIT {} OFFSET {offset}",
+                    if *limit > 1024 { 1024 } else { *limit }
+                )
+            }
+            Self::ManyJson(order_by_top_level_key, limit, offset) => {
+                format!(
+                    "ORDER BY v::jsonb->>'{order_by_top_level_key}' DESC LIMIT {} OFFSET {offset}",
                     if *limit > 1024 { 1024 } else { *limit }
                 )
             }

sql_changes/apps_data_used.sql

@@ -1,2 +1,2 @@
 ALTER TABLE apps
-ADD COLUMN data_used INT NOT NULL DEFAULT 0;
+ADD COLUMN data_used INT NOT NULL DEFAULT 0 CHECK (data_used >= 0);