Merge pull request #114 from dipamsen/crawler-update

Crawler Overhaul

Author: harshkhandeparkar

.github/workflows/test_and_lint.yml

@@ -6,7 +6,7 @@ on:
       - main
   pull_request:
     paths:
-      - 'backend/'
+      - 'backend/**'
 
 
 env:

.gitignore

@@ -6,8 +6,8 @@ iqps.db
 
 crawler/cache
 crawler/qp/*.pdf
-qp.csv
-qp.tar.xz
+qp.json
+qp.tar.gz
 
 go.work.sum
 

README.md

@@ -35,6 +35,10 @@
 
 - [About The Project](#about-the-project)
 - [Development](#development)
+  - [Database](#database)
+  - [Authentication](#authentication)
+    - [OAuth Flow](#oauth-flow)
+  - [Crawler](#crawler)
 - [Deployment](#deployment)
   - [Backend](#backend)
   - [Environment Variables](#environment-variables)
@@ -67,7 +71,8 @@ IQPS was originally created by [Shubham Mishra](https://github.com/grapheo12) in
    - Set up the database (see [Database](#database))
    - Start the Rust backend by running `cargo run .`
 3. Set up the frontend by running `pnpm install` and then `pnpm start` in the `frontend/` directory.
-4. Profit.
+4. (Optional) Set up an http file server for serving static files from the `STATIC_FILE_STORAGE_LOCATION` directory. (eg: `python3 -m http.server 8081`) The host of this server should be set in the `.env` file as `STATIC_FILES_URL`.
+5. Profit.
 
 ### Database
 
@@ -121,7 +126,10 @@ A user is considered as an admin if they are a part of the team `GH_ORG_TEAM_SLU
 
 ### Crawler
 
-[WIP: Steps to locally set up crawler]
+1. Change directory to `crawler/` and run `go mod tidy`.
+2. Run the crawler by running `go run crawler.go`. (Make sure you are connected to the campus network)
+3. This will generate a `qp.tar.gz` file. Transfer this file to the server's `backend/` folder.
+4. In the backend, run `cargo run --bin import-papers` to import the data into the database. (Make sure the database is set up and running)
 
 ## Deployment
 

backend/.gitignore

@@ -1,3 +1,3 @@
 # Added by cargo
 
-/target
+/target

backend/Cargo.lock

@@ -2,6 +2,7 @@
 name = "iqps-backend"
 version = "0.1.0"
 edition = "2021"
+default-run = "iqps-backend"
 
 [dependencies]
 axum = { version = "0.7.7", features = ["multipart"] }
@@ -10,6 +11,7 @@ clap = { version = "4.5.20", features = ["derive", "env"] }
 color-eyre = "0.6.3"
 dotenvy = "0.15.7"
 duplicate = "2.0.0"
+flate2 = "1.0"
 hmac = "0.12.1"
 http = "1.1.0"
 jwt = "0.16.0"
@@ -18,6 +20,8 @@ serde = { version = "1.0.210", features = ["serde_derive"] }
 serde_json = "1.0.128"
 sha2 = "0.10.8"
 sqlx = { version = "0.8.2", features = ["postgres", "runtime-tokio", "chrono"] }
+tar = "0.4"
+tempfile = "3.17.1"
 tokio = { version = "1.40.0", features = ["full"] }
 tower-http = { version = "0.6.1", features = ["cors", "trace"] }
 tracing = "0.1.40"