diff --git a/.gitignore b/.gitignore index a06694b9..224878b5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ go.work.sum frontend/public/pdf.worker.min.mjs *.log -*.patch \ No newline at end of file +*.patch +log \ No newline at end of file diff --git a/README.md b/README.md index 06714e24..30cd36b5 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,12 @@ A user is considered as an admin if they are a part of the team `GH_ORG_TEAM_SLU ### Crawler +The crawler is a go script which crawls and downloads papers from [peqp](http://10.18.24.75/peqp/) (only accessible over campus network) and spits an archive which can be imported into the database. + 1. Change directory to `crawler/` and run `go mod tidy`. -2. Run the crawler by running `go run crawler.go`. (Make sure you are connected to the campus network) +2. Run the crawler by running `go run crawler.go [flags]`. (Make sure you are connected to the campus network) (Run `go run crawler.go -h` to see flags that can be set.) 3. This will generate a `qp.tar.gz` file. Transfer this file to the server's `backend/` folder. -4. (Development): In the backend, run `cargo run --bin import-papers` to import the data into the database. (Make sure the database is set up and running)\ +4. (Development): In the backend, run `cargo run --bin import-papers qp.tar.gz` to import the data into the database. (Make sure the database is set up and running)\ (Production): In the backend, run `./import_papers.sh ./qp.tar.gz` to run the import script in the docker container running the application. ## Deployment diff --git a/backend/import_papers.sh b/backend/import_papers.sh index 29098160..05060a0d 100755 --- a/backend/import_papers.sh +++ b/backend/import_papers.sh @@ -10,10 +10,18 @@ ARCHIVE_PATH="$1" SERVICE="iqps-backend" DEST_PATH="/app/qp.tar.gz" +if [[ ! -f "$ARCHIVE_PATH" ]]; then + echo "Error: File '$ARCHIVE_PATH' not found." + exit 1 +fi + echo "Copying '$ARCHIVE_PATH' to '$SERVICE'..." docker compose cp "$ARCHIVE_PATH" "$SERVICE":"$DEST_PATH" echo "Running import-papers..." docker compose exec "$SERVICE" ./import-papers +echo "Deleting copied file from container..." +docker compose exec "$SERVICE" rm -f "$DEST_PATH" + echo "Done!" diff --git a/backend/src/bin/import-papers.rs b/backend/src/bin/import-papers.rs index 4ea22195..35850f34 100644 --- a/backend/src/bin/import-papers.rs +++ b/backend/src/bin/import-papers.rs @@ -13,22 +13,41 @@ use tempfile::tempdir; use tracing::{info, warn}; use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt}; +#[derive(Parser, Debug)] +#[command( + name = "import-papers", + about = "Imports papers into the database from an archive.", + version, + author +)] +struct Args { + /// Path to the .tar.gz file containing papers (e.g., qp.tar.gz) + file: String, +} + #[tokio::main] async fn main() -> Result<(), Box> { if dotenvy::dotenv().is_ok() { println!("Loaded an existing .env file."); } - let env_vars = env::EnvVars::parse() + + let env_vars = env::EnvVars::parse()? .process() .expect("Failed to parse environment variables"); + let args = Args::parse(); + if !Path::new(&args.file).exists() { + eprintln!("Error: file '{}' not found.", args.file); + std::process::exit(1); + } + let database = db::Database::new(&env_vars) .await .expect("Failed to connect to database"); let dir = tempdir()?; let dir_path = dir.path(); - extract_tar_gz("qp.tar.gz", dir_path)?; + extract_tar_gz(&args.file, dir_path)?; let file = fs::File::open(dir_path.join("qp.json")).expect("Failed to open JSON file"); let reader = BufReader::new(file); @@ -142,9 +161,10 @@ async fn main() -> Result<(), Box> { println!("Finished uploading papers to database."); dir.close()?; + let total_count = database.get_unapproved_papers_count().await?; + let message = format!( - "{} papers have been imported into IQPS!", - count, + "💥 {count} papers have been imported into IQPS!\n\n | Total Unapproved papers: *{total_count}*", ); let _ = slack::send_slack_message( diff --git a/backend/src/env.rs b/backend/src/env.rs index 926d4d8c..d4088a57 100644 --- a/backend/src/env.rs +++ b/backend/src/env.rs @@ -4,95 +4,135 @@ use std::path::PathBuf; -use clap::Parser; use hmac::{digest::InvalidLength, Hmac, Mac}; use sha2::Sha256; use crate::pathutils::Paths; -#[derive(Parser, Clone)] +#[derive(Clone)] pub struct EnvVars { // Database - #[arg(env)] /// Database name pub db_name: String, - #[arg(env)] /// Database hostname pub db_host: String, - #[arg(env)] /// Database port pub db_port: String, - #[arg(env)] /// Database username pub db_user: String, - #[arg(env)] /// Database password pub db_password: String, // Auth - #[arg(env)] /// OAuth app client id (public token) pub gh_client_id: String, - #[arg(env)] /// An org admin's Github token (with the `read:org` permission) pub gh_org_admin_token: String, - #[arg(env)] /// JWT encryption secret (make it a long, randomized string) jwt_secret: String, - #[arg(env)] /// OAuth app client secret pub gh_client_secret: String, - #[arg(env, default_value = "")] /// Github organization name pub gh_org_name: String, - #[arg(env, default_value = "")] /// Github organization team slug (this team has access to admin dashboard) pub gh_org_team_slug: String, - #[arg(env, default_value = "")] /// The usernames of the admins (additional to org team members, comma separated) pub gh_admin_usernames: String, - #[arg(env, default_value = "")] /// URL of Slack webhook for sending notifications pub slack_webhook_url: String, // Other configs - #[arg(env, default_value = "10")] /// Maximum number of papers that can be uploaded at a time pub max_upload_limit: usize, - #[arg(env, default_value = "./log/application.log")] /// Location where logs are stored pub log_location: PathBuf, // Paths - #[arg(env, default_value = "https://static.metakgp.org")] /// The URL of the static files server (odin's vault) static_files_url: String, - #[arg(env, default_value = "/srv/static")] /// The path where static files are served from static_file_storage_location: PathBuf, - #[arg(env, default_value = "/iqps/uploaded")] /// The path where uploaded papers are stored temporarily, relative to the `static_file_storage_location` uploaded_qps_path: PathBuf, - #[arg(env, default_value = "/peqp/qp")] /// The path where library papers (scrapped) are stored, relative to the `static_file_storage_location` library_qps_path: PathBuf, // Server - #[arg(env, default_value = "8080")] /// The port the server listens on pub server_port: i32, // CORS - #[arg(env, default_value = "https://qp.metakgp.org,http://localhost:5173")] /// List of origins allowed (as a list of values separated by commas `origin1, origin2`) pub cors_allowed_origins: String, - #[arg(skip)] /// All paths must be handled using this pub paths: Paths, } -impl EnvVars { +impl EnvVars { + /// Parses the environment variables into the struct + pub fn parse() -> Result> { + let db_name = std::env::var("DB_NAME")?; + let db_host = std::env::var("DB_HOST")?; + let db_port = std::env::var("DB_PORT")?; + let db_user = std::env::var("DB_USER")?; + let db_password = std::env::var("DB_PASSWORD")?; + let gh_client_id = std::env::var("GH_CLIENT_ID")?; + let gh_org_admin_token = std::env::var("GH_ORG_ADMIN_TOKEN")?; + let jwt_secret = std::env::var("JWT_SECRET")?; + let gh_client_secret = std::env::var("GH_CLIENT_SECRET")?; + let gh_org_name = std::env::var("GH_ORG_NAME").unwrap_or_default(); + let gh_org_team_slug = std::env::var("GH_ORG_TEAM_SLUG").unwrap_or_default(); + let gh_admin_usernames = std::env::var("GH_ADMIN_USERNAMES").unwrap_or_default(); + let slack_webhook_url = std::env::var("SLACK_WEBHOOK_URL").unwrap_or_default(); + let max_upload_limit = std::env::var("MAX_UPLOAD_LIMIT") + .unwrap_or_else(|_| "10".to_string()) + .parse::()?; + let log_location = std::env::var("LOG_LOCATION") + .unwrap_or_else(|_| "./log/application.log".to_string()) + .into(); + let static_files_url = std::env::var("STATIC_FILES_URL") + .unwrap_or_else(|_| "https://static.metakgp.org".to_string()); + let static_file_storage_location = std::env::var("STATIC_FILE_STORAGE_LOCATION") + .unwrap_or_else(|_| "/srv/static".to_string()) + .into(); + let uploaded_qps_path = std::env::var("UPLOADED_QPS_PATH") + .unwrap_or_else(|_| "/iqps/uploaded".to_string()) + .into(); + let library_qps_path = std::env::var("LIBRARY_QPS_PATH") + .unwrap_or_else(|_| "/peqp/qp".to_string()) + .into(); + let server_port = std::env::var("SERVER_PORT") + .unwrap_or_else(|_| "8080".to_string()) + .parse::()?; + let cors_allowed_origins = std::env::var("CORS_ALLOWED_ORIGINS") + .unwrap_or_else(|_| "https://qp.metakgp.org,http://localhost:5173".to_string()); + Ok(Self { + db_name, + db_host, + db_port, + db_user, + db_password, + gh_client_id, + gh_org_admin_token, + jwt_secret, + gh_client_secret, + gh_org_name, + gh_org_team_slug, + gh_admin_usernames, + slack_webhook_url, + max_upload_limit, + log_location, + static_files_url, + static_file_storage_location, + uploaded_qps_path, + library_qps_path, + server_port, + cors_allowed_origins, + paths: Paths::default(), + }) + } + /// Processes the environment variables after reading. pub fn process(mut self) -> Result> { self.paths = Paths::new( diff --git a/backend/src/main.rs b/backend/src/main.rs index d39b62da..feb2a5ff 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -2,7 +2,6 @@ //! //! The backend is divided into multiple modules. The [`routing`] module contains all the route handlers and the [`db`] module contains all database queries and models. Other modules are utilities used throughout the backend. -use clap::Parser; use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_subscriber::prelude::*; @@ -22,7 +21,7 @@ async fn main() -> Result<(), Box> { } // Read environment variables - let env_vars = env::EnvVars::parse().process()?; + let env_vars = env::EnvVars::parse()?.process()?; // Initialize logger let (append_writer, _guard) = tracing_appender::non_blocking( diff --git a/frontend/src/components/Common/Common.tsx b/frontend/src/components/Common/Common.tsx index ff047889..092d5c87 100644 --- a/frontend/src/components/Common/Common.tsx +++ b/frontend/src/components/Common/Common.tsx @@ -4,7 +4,10 @@ import './styles/common_styles.scss'; import { IconType } from 'react-icons'; export function Footer() { - return

Made with ❤️ and {""} by MetaKGP

; + return

+ Contribute on GitHub | + Made with ❤️ and {""} by MetaKGP +

; } interface ILinkCommonProps { @@ -46,4 +49,4 @@ export function Header(props: IHeaderProps) { export function Navbar() { return <>; -} \ No newline at end of file +}