Skip to content

Commit f92cdbf

Browse files
DaKheera47DaKheera47
authored
Merge pull request #49 from DaKheera47/formatting-on-push
- don't run docker image build for every PR - Automated Linting: Uses Biome to ensure code quality and consistency across the repository. - Test Orchestration: Executes unit tests for the orchestrator service using Vitest. - Parallel Build Verification: Implements a matrix build strategy to verify orchestrator and extractors simultaneously, reducing total CI runtime. - Optimized Resource Usage: - Concurrency Control: Automatically cancels outdated runs when new commits are pushed. - Selective Script Execution: Skips heavy binary downloads (Camoufox) during extractor builds to speed up verification. - Granular Caching: Configures project-specific dependency caching for faster installation.
2 parents d4e83c0 + c464a77 commit f92cdbf

19 files changed

Lines changed: 869 additions & 603 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
concurrency:
10+
group: ci-${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
lint:
15+
name: Linting (Biome)
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v4
19+
- name: Setup Biome
20+
uses: biomejs/setup-biome@v2
21+
with:
22+
version: 2.3.12
23+
- name: Run Biome
24+
run: biome ci .
25+
26+
test-orchestrator:
27+
name: Orchestrator Tests
28+
runs-on: ubuntu-latest
29+
steps:
30+
- uses: actions/checkout@v4
31+
- name: Setup Node
32+
uses: actions/setup-node@v4
33+
with:
34+
node-version: 20
35+
cache: 'npm'
36+
cache-dependency-path: orchestrator/package-lock.json
37+
- name: Install dependencies
38+
run: npm ci
39+
working-directory: orchestrator
40+
- name: Run Vitest
41+
run: npm run test:run
42+
working-directory: orchestrator
43+
44+
build:
45+
name: Build Verification
46+
runs-on: ubuntu-latest
47+
strategy:
48+
matrix:
49+
project: [orchestrator, extractors/gradcracker, extractors/ukvisajobs]
50+
steps:
51+
- uses: actions/checkout@v4
52+
- name: Setup Node
53+
uses: actions/setup-node@v4
54+
with:
55+
node-version: 20
56+
cache: 'npm'
57+
cache-dependency-path: ${{ matrix.project }}/package-lock.json
58+
59+
- name: Build ${{ matrix.project }}
60+
run: |
61+
if [[ "${{ matrix.project }}" == extractors/* ]]; then
62+
npm ci --ignore-scripts
63+
else
64+
npm ci
65+
fi
66+
npm run build
67+
working-directory: ${{ matrix.project }}

‎.github/workflows/ghcr.yml‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
# build and push releases to ghcr
2-
# build for PRs only to test failures
3-
42
name: build-and-push-ghcr
53

64
on:
75
push:
86
tags: ["v*"]
9-
pull_request:
107

118
permissions:
129
contents: read
@@ -30,7 +27,6 @@ jobs:
3027
uses: docker/setup-buildx-action@v3
3128

3229
- name: Log in to GHCR
33-
if: github.event_name != 'pull_request'
3430
uses: docker/login-action@v3
3531
with:
3632
registry: ghcr.io
@@ -48,21 +44,7 @@ jobs:
4844
# Optional: also publish :latest for version tags
4945
type=raw,value=latest
5046
51-
- name: Build (PR)
52-
if: github.event_name == 'pull_request'
53-
uses: docker/build-push-action@v6
54-
with:
55-
context: .
56-
file: ./Dockerfile
57-
push: false
58-
platforms: linux/amd64,linux/arm64
59-
tags: ${{ steps.meta.outputs.tags }}
60-
labels: ${{ steps.meta.outputs.labels }}
61-
cache-from: type=gha
62-
cache-to: type=gha,mode=max
63-
64-
- name: Build and push (tag)
65-
if: github.event_name != 'pull_request'
47+
- name: Build and push
6648
uses: docker/build-push-action@v6
6749
with:
6850
context: .

‎biome.json‎

Lines changed: 25 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,27 @@
11
{
2-
"$schema": "https://biomejs.dev/schemas/2.3.12/schema.json",
3-
"formatter": {
4-
"indentStyle": "space",
5-
"indentWidth": 2
6-
},
7-
"files": {
8-
"includes": [
9-
"**",
10-
"!!**/dist"
11-
]
12-
},
13-
"css": {
14-
"parser": {
15-
"tailwindDirectives": true
2+
"$schema": "https://biomejs.dev/schemas/2.3.12/schema.json",
3+
"formatter": {
4+
"indentStyle": "space",
5+
"indentWidth": 2
6+
},
7+
"files": {
8+
"includes": ["**", "!!**/dist"]
9+
},
10+
"css": {
11+
"parser": {
12+
"tailwindDirectives": true
13+
}
14+
},
15+
"overrides": [
16+
{
17+
"includes": ["**/*.test.ts", "**/*.test.tsx", "**/test-utils.ts"],
18+
"linter": {
19+
"rules": {
20+
"suspicious": {
21+
"noExplicitAny": "off"
22+
}
1623
}
17-
},
18-
"overrides": [
19-
{
20-
"includes": [
21-
"**/*.test.ts",
22-
"**/*.test.tsx",
23-
"**/test-utils.ts"
24-
],
25-
"linter": {
26-
"rules": {
27-
"suspicious": {
28-
"noExplicitAny": "off"
29-
}
30-
}
31-
}
32-
}
33-
]
34-
}
24+
}
25+
}
26+
]
27+
}

‎extractors/gradcracker/package.json‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
11
{
2-
"name": "job-flow",
3-
"version": "0.0.1",
4-
"type": "module",
5-
"description": "This is an example of a Crawlee project.",
6-
"dependencies": {
7-
"camoufox-js": "^0.8.0",
8-
"crawlee": "^3.0.0",
9-
"playwright": "*"
10-
},
11-
"devDependencies": {
12-
"@apify/tsconfig": "^0.1.0",
13-
"@types/fs-extra": "^11",
14-
"@types/node": "^24.0.0",
15-
"fs-extra": "^11.3.0",
16-
"tsx": "^4.4.0",
17-
"typescript": "~5.9.0"
18-
},
19-
"scripts": {
20-
"start": "npm run start:dev",
21-
"start:prod": "node dist/main.js",
22-
"start:dev": "tsx src/main.ts",
23-
"build": "tsc",
24-
"test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
25-
"get-binaries": "camoufox-js fetch",
26-
"postinstall": "npm run get-binaries"
27-
},
28-
"author": "It's not you it's me",
29-
"license": "ISC"
2+
"name": "job-flow",
3+
"version": "0.0.1",
4+
"type": "module",
5+
"description": "This is an example of a Crawlee project.",
6+
"dependencies": {
7+
"camoufox-js": "^0.8.0",
8+
"crawlee": "^3.0.0",
9+
"playwright": "*"
10+
},
11+
"devDependencies": {
12+
"@apify/tsconfig": "^0.1.0",
13+
"@types/fs-extra": "^11",
14+
"@types/node": "^24.0.0",
15+
"fs-extra": "^11.3.0",
16+
"tsx": "^4.4.0",
17+
"typescript": "~5.9.0"
18+
},
19+
"scripts": {
20+
"start": "npm run start:dev",
21+
"start:prod": "node dist/main.js",
22+
"start:dev": "tsx src/main.ts",
23+
"build": "tsc",
24+
"test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
25+
"get-binaries": "camoufox-js fetch",
26+
"postinstall": "npm run get-binaries"
27+
},
28+
"author": "It's not you it's me",
29+
"license": "ISC"
3030
}

‎extractors/gradcracker/src/main.ts‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@
22
import { launchOptions } from "camoufox-js";
33
import { PlaywrightCrawler } from "crawlee";
44
import { firefox } from "playwright";
5-
6-
import { router } from "./routes.js";
75
import { initJobOpsProgress } from "./progress.js";
6+
import { router } from "./routes.js";
87

98
// locations
109
const locations = [
@@ -17,10 +16,7 @@ const locations = [
1716
];
1817

1918
// roles
20-
const defaultRoles = [
21-
"web-development",
22-
"software-systems",
23-
];
19+
const defaultRoles = ["web-development", "software-systems"];
2420

2521
let roles = defaultRoles;
2622
const envRolesRaw = process.env.GRADCRACKER_SEARCH_TERMS;
@@ -29,15 +25,16 @@ if (envRolesRaw) {
2925
try {
3026
const parsed = JSON.parse(envRolesRaw) as string[];
3127
if (Array.isArray(parsed) && parsed.length > 0) {
32-
roles = parsed.map(term =>
33-
term.toLowerCase()
34-
.replace(/[^a-z0-9]+/g, '-')
35-
.replace(/^-+|-+$/g, '')
28+
roles = parsed.map((term) =>
29+
term
30+
.toLowerCase()
31+
.replace(/[^a-z0-9]+/g, "-")
32+
.replace(/^-+|-+$/g, ""),
3633
);
37-
console.log(`Using configured search terms: ${roles.join(', ')}`);
34+
console.log(`Using configured search terms: ${roles.join(", ")}`);
3835
}
3936
} catch (e) {
40-
console.warn('Failed to parse GRADCRACKER_SEARCH_TERMS', e);
37+
console.warn("Failed to parse GRADCRACKER_SEARCH_TERMS", e);
4138
}
4239
}
4340

@@ -46,12 +43,12 @@ const gradcrackerUrls = locations.flatMap((location) => {
4643
return roles.map((role) => {
4744
return {
4845
url: `https://www.gradcracker.com/search/computing-technology/${role}-graduate-jobs-in-${location}?order=dateAdded`,
49-
role
46+
role,
5047
};
5148
});
5249
});
5350

54-
console.log(`Total gradcracker URLs: ${gradcrackerUrls.length}`)
51+
console.log(`Total gradcracker URLs: ${gradcrackerUrls.length}`);
5552

5653
const startUrls = gradcrackerUrls.map(({ url, role }) => ({
5754
url,

‎extractors/gradcracker/src/progress.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ interface JobOpsCrawlProgressState {
2626
const PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
2727
const isEnabled = () => process.env.JOBOPS_EMIT_PROGRESS === "1";
2828

29-
let state: JobOpsCrawlProgressState = {
29+
const state: JobOpsCrawlProgressState = {
3030
listPagesProcessed: 0,
3131
jobCardsFound: 0,
3232
jobPagesEnqueued: 0,
@@ -80,4 +80,3 @@ export function markJobPageDone(params: { currentUrl: string }): void {
8080
state.currentUrl = params.currentUrl;
8181
emit();
8282
}
83-

0 commit comments

Comments
 (0)