t/tetratto
1
0
Fork 0
Code Issues Pull requests Projects Releases 9 Packages Wiki Activity Actions

add: move upload server to buckets

Browse source
This commit is contained in:
trisua 2025-08-21 00:30:29 -04:00
parent 8116307ba0
commit 75fe720f21
83 changed files with 351 additions and 458 deletions
Download patch file Download diff file Expand all files Collapse all files

31
crates/core/src/config.rs
View file

@ -257,6 +257,24 @@ impl Default for ManualsConfig {
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct ServiceHostsConfig {
/// Buckets host <https://trisua.com/t/buckets>.
pub buckets: String,
/// Littleweb browser host.
#[serde(default)]
pub littleweb: String,
}
impl Default for ServiceHostsConfig {
fn default() -> Self {
Self {
buckets: String::new(),
littleweb: String::new(),
}
}
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
pub enum StringBan {
/// An exact string.
@ -298,10 +316,9 @@ pub struct Config {
/// so this host should be included in there as well.
#[serde(default = "default_host")]
pub host: String,
/// The main public host of the littleweb server. **Not** used to check against banned hosts,
/// so this host should be included in there as well.
#[serde(default = "default_lw_host")]
pub lw_host: String,
/// The main public host of the required microservices.
#[serde(default = "default_service_hosts")]
pub service_hosts: ServiceHostsConfig,
/// Database security.
#[serde(default = "default_security")]
pub security: SecurityConfig,
@ -382,8 +399,8 @@ fn default_host() -> String {
String::new()
}
fn default_lw_host() -> String {
String::new()
fn default_service_hosts() -> ServiceHostsConfig {
ServiceHostsConfig::default()
}
fn default_security() -> SecurityConfig {
@ -459,7 +476,7 @@ impl Default for Config {
port: default_port(),
banned_hosts: default_banned_hosts(),
host: default_host(),
lw_host: default_lw_host(),
service_hosts: default_service_hosts(),
database: default_database(),
security: default_security(),
dirs: default_dirs(),

4
crates/core/src/database/ads.rs
View file

@ -143,7 +143,9 @@ impl DataManager {
}
// remove upload
self.delete_upload(ad.upload_id).await?;
if let Err(e) = self.2.delete_upload(ad.upload_id).await {
return Err(Error::MiscError(e.to_string()));
}
// ...
let conn = match self.0.connect().await {

9
crates/core/src/database/auth.rs
View file

@ -648,8 +648,13 @@ impl DataManager {
}
// delete uploads
for upload in self.get_uploads_by_owner_all(user.id).await? {
self.delete_upload(upload.id).await?;
for upload in match self.2.get_uploads_by_owner_all(user.id).await {
Ok(x) => x,
Err(e) => return Err(Error::MiscError(e.to_string())),
} {
if let Err(e) = self.2.delete_upload(upload.id).await {
return Err(Error::MiscError(e.to_string()));
}
}
// delete polls

2
crates/core/src/database/common.rs
View file

@ -28,7 +28,6 @@ impl DataManager {
execute!(&conn, common::CREATE_TABLE_IPBLOCKS).unwrap();
execute!(&conn, common::CREATE_TABLE_CHANNELS).unwrap();
execute!(&conn, common::CREATE_TABLE_MESSAGES).unwrap();
execute!(&conn, common::CREATE_TABLE_UPLOADS).unwrap();
execute!(&conn, common::CREATE_TABLE_EMOJIS).unwrap();
execute!(&conn, common::CREATE_TABLE_STACKS).unwrap();
execute!(&conn, common::CREATE_TABLE_DRAFTS).unwrap();
@ -61,6 +60,7 @@ impl DataManager {
.set("atto.active_connections:chats".to_string(), "0".to_string())
.await;
self.2.init().await.expect("failed to init buckets manager");
Ok(())
}

1
crates/core/src/database/drivers/common.rs
View file

@ -16,7 +16,6 @@ pub const CREATE_TABLE_QUESTIONS: &str = include_str!("./sql/create_questions.sq
pub const CREATE_TABLE_IPBLOCKS: &str = include_str!("./sql/create_ipblocks.sql");
pub const CREATE_TABLE_CHANNELS: &str = include_str!("./sql/create_channels.sql");
pub const CREATE_TABLE_MESSAGES: &str = include_str!("./sql/create_messages.sql");
pub const CREATE_TABLE_UPLOADS: &str = include_str!("./sql/create_uploads.sql");
pub const CREATE_TABLE_EMOJIS: &str = include_str!("./sql/create_emojis.sql");
pub const CREATE_TABLE_STACKS: &str = include_str!("./sql/create_stacks.sql");
pub const CREATE_TABLE_DRAFTS: &str = include_str!("./sql/create_drafts.sql");

20
crates/core/src/database/drivers/mod.rs
View file

@ -4,13 +4,29 @@ use std::collections::HashMap;
use tetratto_l10n::{read_langs, LangFile};
use oiseau::postgres::{DataManager as OiseauManager, Result};
use crate::config::Config;
use buckets_core::{DataManager as BucketsManager, Config as BucketsConfig};
#[derive(Clone)]
pub struct DataManager(pub OiseauManager<Config>, pub HashMap<String, LangFile>);
pub struct DataManager(
pub OiseauManager<Config>,
pub HashMap<String, LangFile>,
pub BucketsManager,
);
impl DataManager {
/// Create a new [`DataManager`].
pub async fn new(config: Config) -> Result<Self> {
Ok(Self(OiseauManager::new(config).await?, read_langs()))
let buckets_manager = BucketsManager::new(BucketsConfig {
directory: format!("{}/{}", config.dirs.media, "uploads"),
database: config.database.clone(),
})
.await
.expect("failed to create buckets manager");
Ok(Self(
OiseauManager::new(config).await?,
read_langs(),
buckets_manager,
))
}
}

7
crates/core/src/database/drivers/sql/create_uploads.sql
View file

@ -1,7 +0,0 @@
CREATE TABLE IF NOT EXISTS uploads (
id BIGINT NOT NULL PRIMARY KEY,
created BIGINT NOT NULL,
owner BIGINT NOT NULL,
what TEXT NOT NULL,
alt TEXT NOT NULL
)

4
crates/core/src/database/emojis.rs
View file

@ -194,7 +194,9 @@ impl DataManager {
}
// delete upload
self.delete_upload(emoji.upload_id).await?;
if let Err(e) = self.2.delete_upload(emoji.upload_id).await {
return Err(Error::MiscError(e.to_string()));
}
// ...
self.0.1.remove(format!("atto.emoji:{}", id)).await;

1
crates/core/src/database/mod.rs
View file

@ -33,7 +33,6 @@ mod services;
mod stackblocks;
mod stacks;
mod transfers;
mod uploads;
mod user_warnings;
mod userblocks;
mod userfollows;

8
crates/core/src/database/posts.rs
View file

@ -2262,7 +2262,9 @@ impl DataManager {
// delete uploads
for upload in y.uploads {
self.delete_upload(upload).await?;
if let Err(e) = self.2.delete_upload(upload).await {
return Err(Error::MiscError(e.to_string()));
}
}
// remove poll
@ -2356,7 +2358,9 @@ impl DataManager {
// delete uploads
for upload in y.uploads {
self.delete_upload(upload).await?;
if let Err(e) = self.2.delete_upload(upload).await {
return Err(Error::MiscError(e.to_string()));
}
}
// delete question (if not global question)

8
crates/core/src/database/products.rs
View file

@ -252,11 +252,15 @@ If your product is a purchase of goods or services, please be sure to fulfill th
// remove uploads
for upload in product.uploads.thumbnails {
self.delete_upload(upload).await?;
if let Err(e) = self.2.delete_upload(upload).await {
return Err(Error::MiscError(e.to_string()));
};
}
if product.uploads.reward != 0 {
self.delete_upload(product.uploads.reward).await?;
if let Err(e) = self.2.delete_upload(product.uploads.reward).await {
return Err(Error::MiscError(e.to_string()));
}
}
// ...

36
crates/core/src/database/questions.rs
View file

@ -4,7 +4,7 @@ use tetratto_shared::unix_epoch_timestamp;
use crate::model::addr::RemoteAddr;
use crate::model::communities::Post;
use crate::model::communities_permissions::CommunityPermission;
use crate::model::uploads::{MediaType, MediaUpload};
use buckets_core::model::{MediaType, MediaUpload};
use crate::model::{
Error, Result,
communities::Question,
@ -463,9 +463,18 @@ impl DataManager {
for _ in 0..drawings.len() {
data.drawings.push(
self.create_upload(MediaUpload::new(MediaType::Carpgraph, data.owner))
.await?
.id,
match self
.2
.create_upload(MediaUpload::new(
MediaType::Carpgraph,
data.owner,
"drawings".to_string(),
))
.await
{
Ok(x) => x.id,
Err(_) => continue,
},
);
}
@ -516,14 +525,23 @@ impl DataManager {
let drawing = match drawings.get(i) {
Some(d) => d,
None => {
self.delete_upload(*drawing_id).await?;
if let Err(e) = self.2.delete_upload(*drawing_id).await {
return Err(Error::MiscError(e.to_string()));
}
continue;
}
};
let upload = self.get_upload_by_id(*drawing_id).await?;
let upload = match self.2.get_upload_by_id(*drawing_id).await {
Ok(x) => x,
Err(e) => return Err(Error::MiscError(e.to_string())),
};
if let Err(e) = std::fs::write(&upload.path(&self.0.0).to_string(), drawing.to_vec()) {
if let Err(e) = std::fs::write(
&upload.path(&self.2.0.0.directory).to_string(),
drawing.to_vec(),
) {
return Err(Error::MiscError(e.to_string()));
}
}
@ -595,7 +613,9 @@ impl DataManager {
// delete uploads
for upload in y.drawings {
self.delete_upload(upload).await?;
if let Err(e) = self.2.delete_upload(upload).await {
return Err(Error::MiscError(e.to_string()));
}
}
// return

194
crates/core/src/database/uploads.rs
View file

@ -1,194 +0,0 @@
use oiseau::cache::Cache;
use crate::model::auth::User;
use crate::model::permissions::FinePermission;
use crate::model::{Error, Result, uploads::MediaUpload};
use crate::{auto_method, DataManager};
use oiseau::PostgresRow;
use oiseau::{execute, get, query_rows, params};
impl DataManager {
/// Get a [`MediaUpload`] from an SQL row.
pub(crate) fn get_upload_from_row(x: &PostgresRow) -> MediaUpload {
MediaUpload {
id: get!(x->0(i64)) as usize,
created: get!(x->1(i64)) as usize,
owner: get!(x->2(i64)) as usize,
what: serde_json::from_str(&get!(x->3(String))).unwrap(),
alt: get!(x->4(String)),
}
}
auto_method!(get_upload_by_id(usize as i64)@get_upload_from_row -> "SELECT * FROM uploads WHERE id = $1" --name="upload" --returns=MediaUpload --cache-key-tmpl="atto.upload:{}");
/// Get all uploads (paginated).
///
/// # Arguments
/// * `batch` - the limit of items in each page
/// * `page` - the page number
pub async fn get_uploads(&self, batch: usize, page: usize) -> Result<Vec<MediaUpload>> {
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = query_rows!(
&conn,
"SELECT * FROM uploads ORDER BY created DESC LIMIT $1 OFFSET $2",
&[&(batch as i64), &((page * batch) as i64)],
|x| { Self::get_upload_from_row(x) }
);
if res.is_err() {
return Err(Error::GeneralNotFound("upload".to_string()));
}
Ok(res.unwrap())
}
/// Get all uploads by their owner (paginated).
///
/// # Arguments
/// * `owner` - the ID of the owner of the upload
/// * `batch` - the limit of items in each page
/// * `page` - the page number
pub async fn get_uploads_by_owner(
&self,
owner: usize,
batch: usize,
page: usize,
) -> Result<Vec<MediaUpload>> {
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = query_rows!(
&conn,
"SELECT * FROM uploads WHERE owner = $1 ORDER BY created DESC LIMIT $2 OFFSET $3",
&[&(owner as i64), &(batch as i64), &((page * batch) as i64)],
|x| { Self::get_upload_from_row(x) }
);
if res.is_err() {
return Err(Error::GeneralNotFound("upload".to_string()));
}
Ok(res.unwrap())
}
/// Get all uploads by their owner.
///
/// # Arguments
/// * `owner` - the ID of the owner of the upload
pub async fn get_uploads_by_owner_all(&self, owner: usize) -> Result<Vec<MediaUpload>> {
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = query_rows!(
&conn,
"SELECT * FROM uploads WHERE owner = $1 ORDER BY created DESC",
&[&(owner as i64)],
|x| { Self::get_upload_from_row(x) }
);
if res.is_err() {
return Err(Error::GeneralNotFound("upload".to_string()));
}
Ok(res.unwrap())
}
/// Create a new upload in the database.
///
/// # Arguments
/// * `data` - a mock [`MediaUpload`] object to insert
pub async fn create_upload(&self, data: MediaUpload) -> Result<MediaUpload> {
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = execute!(
&conn,
"INSERT INTO uploads VALUES ($1, $2, $3, $4, $5)",
params![
&(data.id as i64),
&(data.created as i64),
&(data.owner as i64),
&serde_json::to_string(&data.what).unwrap().as_str(),
&data.alt,
]
);
if let Err(e) = res {
return Err(Error::DatabaseError(e.to_string()));
}
// return
Ok(data)
}
pub async fn delete_upload(&self, id: usize) -> Result<()> {
// if !user.permissions.check(FinePermission::MANAGE_UPLOADS) {
// return Err(Error::NotAllowed);
// }
// delete file
// it's most important that the file gets off the file system first, even
// if there's an issue in the database
//
// the actual file takes up much more space than the database entry.
let upload = self.get_upload_by_id(id).await?;
upload.remove(&self.0.0)?;
// delete from database
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = execute!(&conn, "DELETE FROM uploads WHERE id = $1", &[&(id as i64)]);
if let Err(e) = res {
return Err(Error::DatabaseError(e.to_string()));
}
self.0.1.remove(format!("atto.upload:{}", id)).await;
// return
Ok(())
}
pub async fn delete_upload_checked(&self, id: usize, user: &User) -> Result<()> {
let upload = self.get_upload_by_id(id).await?;
// check user permission
if user.id != upload.owner && !user.permissions.check(FinePermission::MANAGE_UPLOADS) {
return Err(Error::NotAllowed);
}
// delete file
upload.remove(&self.0.0)?;
// ...
let conn = match self.0.connect().await {
Ok(c) => c,
Err(e) => return Err(Error::DatabaseConnection(e.to_string())),
};
let res = execute!(&conn, "DELETE FROM uploads WHERE id = $1", &[&(id as i64)]);
if let Err(e) = res {
return Err(Error::DatabaseError(e.to_string()));
}
self.0.1.remove(format!("atto.upload:{}", id)).await;
Ok(())
}
auto_method!(update_upload_alt(&str)@get_upload_by_id:FinePermission::MANAGE_UPLOADS; -> "UPDATE uploads SET alt = $1 WHERE id = $2" --cache-key-tmpl="atto.upload:{}");
}

1
crates/core/src/model/mod.rs
View file

@ -19,7 +19,6 @@ pub mod stacks;
pub mod uploads;
use std::fmt::Display;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]

89
crates/core/src/model/uploads.rs
View file

@ -1,93 +1,6 @@
use pathbufd::PathBufD;
use serde::{Serialize, Deserialize};
use tetratto_shared::{snow::Snowflake, unix_epoch_timestamp};
use crate::config::Config;
use std::fs::{write, exists, remove_file};
use super::{Error, Result};
#[derive(Serialize, Deserialize, PartialEq, Eq)]
pub enum MediaType {
#[serde(alias = "image/webp")]
Webp,
#[serde(alias = "image/avif")]
Avif,
#[serde(alias = "image/png")]
Png,
#[serde(alias = "image/jpg")]
Jpg,
#[serde(alias = "image/gif")]
Gif,
#[serde(alias = "image/carpgraph")]
Carpgraph,
}
impl MediaType {
pub fn extension(&self) -> &str {
match self {
Self::Webp => "webp",
Self::Avif => "avif",
Self::Png => "png",
Self::Jpg => "jpg",
Self::Gif => "gif",
Self::Carpgraph => "carpgraph",
}
}
pub fn mime(&self) -> String {
format!("image/{}", self.extension())
}
}
#[derive(Serialize, Deserialize)]
pub struct MediaUpload {
pub id: usize,
pub created: usize,
pub owner: usize,
pub what: MediaType,
pub alt: String,
}
impl MediaUpload {
/// Create a new [`MediaUpload`].
pub fn new(what: MediaType, owner: usize) -> Self {
Self {
id: Snowflake::new().to_string().parse::<usize>().unwrap(),
created: unix_epoch_timestamp(),
owner,
what,
alt: String::new(),
}
}
/// Get the path to the fs file for this upload.
pub fn path(&self, config: &Config) -> PathBufD {
PathBufD::current()
.extend(&[config.dirs.media.as_str(), "uploads"])
.join(format!("{}.{}", self.id, self.what.extension()))
}
/// Write to this upload in the file system.
pub fn write(&self, config: &Config, bytes: &[u8]) -> Result<()> {
match write(self.path(config), bytes) {
Ok(_) => Ok(()),
Err(e) => Err(Error::MiscError(e.to_string())),
}
}
/// Delete this upload in the file system.
pub fn remove(&self, config: &Config) -> Result<()> {
let path = self.path(config);
if !exists(&path).unwrap() {
return Ok(());
}
match remove_file(path) {
Ok(_) => Ok(()),
Err(e) => Err(Error::MiscError(e.to_string())),
}
}
}
pub use buckets_core::model::*;
#[derive(Serialize, Deserialize)]
pub struct CustomEmoji {