Merge pull request #47 from ITSEC-Research/tomi-dev

Tomi dev

Author: mastomii

.env.example

@@ -9,6 +9,14 @@
 #   3. docker-compose up -d --build
 # =====================================================
 
+# -----------------------------------------------------------
+# AUTHENTICATION
+# -----------------------------------------------------------
+# JWT secret for signing authentication tokens
+# IMPORTANT: Generate a strong random secret (minimum 32 characters)
+# Example: openssl rand -hex 32
+JWT_SECRET=super_secret_random_string_at_least_32_chars_long
+
 # -----------------------------------------------------------
 # MAIN DATABASE (MySQL/MariaDB)
 # -----------------------------------------------------------
@@ -41,6 +49,15 @@ CLICKHOUSE_DB=bronvault_analytics
 SYNC_USER=clickhouse_sync
 SYNC_PASSWORD=change_me_sync_password_123
 
+# -----------------------------------------------------------
+# OBJECT STORAGE (MinIO - S3 Compatible)
+# -----------------------------------------------------------
+# MinIO root credentials for the admin console
+# Console accessible at http://localhost:9002
+# S3 API accessible at http://localhost:9001
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=minioadmin
+
 # =====================================================
 # IMPORTANT SECURITY NOTES:
 # =====================================================

.gitignore

@@ -7,6 +7,7 @@ uploads/*
 error.log
 clickhouse-data
 mysql-data
+minio-data
 tmp
 .DS_Store
 .devtasks

app/api/auth/change-password/route.ts

@@ -3,6 +3,7 @@ import { pool } from "@/lib/mysql";
 import bcrypt from "bcryptjs";
 import type { RowDataPacket } from "mysql2";
 import { validateRequest } from "@/lib/auth";
+import { passwordSchema } from "@/lib/validation";
 
 export async function POST(request: NextRequest) {
   const { currentPassword, newPassword } = await request.json();
@@ -14,6 +15,15 @@ export async function POST(request: NextRequest) {
     return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 });
   }
 
+  // SECURITY: Validate new password strength (MED-01)
+  const passwordValidation = passwordSchema.safeParse(newPassword);
+  if (!passwordValidation.success) {
+    return NextResponse.json(
+      { success: false, error: passwordValidation.error.errors.map(e => e.message).join(", ") },
+      { status: 400 }
+    );
+  }
+
   try {
     // Get current user data
     const [users] = await pool.query<RowDataPacket[]>(

app/api/auth/login/route.ts

@@ -65,17 +65,26 @@ export async function POST(request: NextRequest) {
       // This token is short-lived (5 minutes) and can only be used for 2FA verification
       const pending2FAToken = await generatePending2FAToken(String(user.id))
 
-      // Return requires2FA flag with secure pending token - don't issue auth token yet
-      return NextResponse.json({
+      // SECURITY: Store pending2FAToken in httpOnly cookie instead of response body (HIGH-02)
+      const response = NextResponse.json({
         success: true,
         requires2FA: true,
-        pending2FAToken, // Secure token that proves password was verified
         message: "Please enter your 2FA code"
       })
+
+      response.cookies.set("pending_2fa", pending2FAToken, {
+        httpOnly: true,
+        secure: process.env.NODE_ENV === 'production',
+        sameSite: 'strict',
+        path: '/api/auth/verify-totp',
+        maxAge: 300, // 5 minutes
+      })
+
+      return response
     }
 
-    // Get user role - default to 'admin' for backwards compatibility
-    const userRole: UserRole = user.role || 'admin'
+    // Get user role - default to 'analyst' for least privilege
+    const userRole: UserRole = user.role || 'analyst'
 
     // Generate JWT token with role included
     const token = await generateToken({

app/api/auth/register-first-user/route.ts

@@ -27,11 +27,11 @@ export async function POST(request: NextRequest) {
       }, { status: 400 })
     }
 
-    // Validate password strength - matches validation.ts requirements
-    if (password.length < 8) {
+    // Validate password strength - matches validation.ts requirements (LOW-01)
+    if (password.length < 12) {
       return NextResponse.json({ 
         success: false, 
-        error: "Password must be at least 8 characters long" 
+        error: "Password must be at least 12 characters long" 
       }, { status: 400 })
     }
 
@@ -54,6 +54,12 @@ export async function POST(request: NextRequest) {
         error: "Password must contain at least one number" 
       }, { status: 400 })
     }
+    if (!/[^A-Za-z0-9]/.test(password)) {
+      return NextResponse.json({ 
+        success: false, 
+        error: "Password must contain at least one special character" 
+      }, { status: 400 })
+    }
 
     // Check if users table exists, create if not
     const [tables] = await pool.query<RowDataPacket[]>(
@@ -83,41 +89,27 @@ export async function POST(request: NextRequest) {
     }
 
     // SECURITY CHECK: Ensure no users exist before allowing registration
-    const [existingUsers] = await pool.query<RowDataPacket[]>(
-      "SELECT COUNT(*) as count FROM users"
+    // Use atomic INSERT...SELECT to prevent race condition (MED-05)
+    // If another request created a user between our check and insert, this will insert 0 rows
+    const hashedPassword = await bcrypt.hash(password, 12)
+
+    const [insertResult] = await pool.query<RowDataPacket[]>(
+      `INSERT INTO users (email, password_hash, name, role)
+       SELECT ?, ?, ?, 'admin'
+       FROM DUAL
+       WHERE NOT EXISTS (SELECT 1 FROM users LIMIT 1)`,
+      [email, hashedPassword, name]
     )
 
-    const userCount = Array.isArray(existingUsers) && existingUsers.length > 0 ? existingUsers[0].count : 0
+    const affectedRows = (insertResult as any).affectedRows || 0
 
-    if (userCount > 0) {
+    if (affectedRows === 0) {
       return NextResponse.json({ 
         success: false, 
         error: "Registration is only allowed when no users exist. Please use login instead." 
       }, { status: 403 })
     }
 
-    // Check if email already exists (extra safety)
-    const [emailCheck] = await pool.query<RowDataPacket[]>(
-      "SELECT id FROM users WHERE email = ? LIMIT 1",
-      [email]
-    )
-
-    if (Array.isArray(emailCheck) && emailCheck.length > 0) {
-      return NextResponse.json({ 
-        success: false, 
-        error: "Email already exists" 
-      }, { status: 400 })
-    }
-
-    // Hash password
-    const hashedPassword = await bcrypt.hash(password, 12)
-
-    // Create first user - always admin role for first user
-    await pool.query(
-      "INSERT INTO users (email, password_hash, name, role) VALUES (?, ?, ?, 'admin')",
-      [email, hashedPassword, name]
-    )
-
     console.log("✅ First user created successfully:", email)
 
     return NextResponse.json({ 

app/api/auth/verify-totp/route.ts

@@ -16,7 +16,10 @@ import type { RowDataPacket } from "mysql2"
 import { logUserAction } from "@/lib/audit-log"
 
 export async function POST(request: NextRequest) {
-  const { pending2FAToken, code, isBackupCode } = await request.json()
+  const { code, isBackupCode } = await request.json()
+
+  // SECURITY: Read pending2FAToken from httpOnly cookie instead of request body (HIGH-02)
+  const pending2FAToken = request.cookies.get('pending_2fa')?.value
 
   // SECURITY: Validate pending 2FA token instead of accepting userId directly
   // This ensures the user has already passed password authentication
@@ -95,7 +98,7 @@ export async function POST(request: NextRequest) {
     }
 
     // Generate JWT token
-    const userRole: UserRole = userData.role || 'admin'
+    const userRole: UserRole = userData.role || 'analyst'
     const token = await generateToken({
       userId: String(userData.id),
       username: userData.name || userData.email,
@@ -125,6 +128,15 @@ export async function POST(request: NextRequest) {
 
     response.cookies.set("auth", token, getSecureCookieOptions())
 
+    // SECURITY: Clear the pending_2fa cookie after successful verification
+    response.cookies.set("pending_2fa", "", {
+      httpOnly: true,
+      secure: process.env.NODE_ENV === 'production',
+      sameSite: 'strict',
+      path: '/api/auth/verify-totp',
+      maxAge: 0,
+    })
+
     return response
   } catch (err) {
     console.error("Verify TOTP error:", err)

app/api/browser-analysis/route.ts

@@ -57,6 +57,7 @@ export async function GET(request: NextRequest) {
     // Build date filter - credentials are linked to devices, so we filter by device_id
     // Get device_ids first, then use them in the query
     let deviceFilter = ""
+    let deviceFilterParams: Record<string, unknown> = {}
     if (hasDateFilter) {
       const { whereClause: deviceDateFilter } = buildDeviceDateFilter(dateFilter)
       const { whereClause: systemInfoDateFilter } = buildSystemInfoDateFilter(dateFilter)
@@ -86,9 +87,9 @@ export async function GET(request: NextRequest) {
         return NextResponse.json({ success: true, browserAnalysis: [] })
       }
 
-      // Use array format for ClickHouse IN clause
-      const deviceIdsStr = deviceIds.map(id => `'${id.replace(/'/g, "''")}'`).join(', ')
-      deviceFilter = `AND device_id IN (${deviceIdsStr})`
+      // SECURITY: Use parameterized array for ClickHouse IN clause (CRIT-09)
+      deviceFilter = `AND device_id IN {filterDeviceIds:Array(String)}`
+      deviceFilterParams = { filterDeviceIds: deviceIds }
     }
 
     // Get all (device_id, browser) pairs to properly count unique devices per normalized browser
@@ -99,7 +100,7 @@ export async function GET(request: NextRequest) {
       FROM credentials 
       WHERE browser IS NOT NULL AND browser != '' ${deviceFilter}`
 
-    const results = await executeClickHouseQuery(baseQuery) as any[];
+    const results = await executeClickHouseQuery(baseQuery, deviceFilterParams) as any[];
 
     if (!Array.isArray(results)) {
       return NextResponse.json({ success: false, error: "Invalid data format" }, { status: 500 });

app/api/domain-recon/credentials/route.ts

@@ -2,6 +2,8 @@ import { NextRequest, NextResponse } from "next/server"
 import { executeQuery as executeClickHouseQuery } from "@/lib/clickhouse"
 import { validateRequest } from "@/lib/auth"
 import { throwIfAborted, getRequestSignal, handleAbortError } from "@/lib/api-helpers"
+import { parseSearchQuery } from "@/lib/query-parser"
+import { buildDomainReconCondition, buildKeywordReconCondition } from "@/lib/search-query-builder"
 
 export async function POST(request: NextRequest) {
   // ✅ Check abort VERY EARLY - before validateRequest
@@ -24,11 +26,13 @@ export async function POST(request: NextRequest) {
     }
 
     // Normalize Domain
-    let cleanDomain = targetDomain.trim().toLowerCase()
-    cleanDomain = cleanDomain.replace(/^https?:\/\//, '').replace(/^www\./, '').split('/')[0].split(':')[0]
+    const cleanDomain = targetDomain.trim().toLowerCase()
+
+    // Parse query for operator support (OR, NOT, wildcard, exact)
+    const parsed = parseSearchQuery(cleanDomain)
 
     // Cleaner log: only show search if present
-    const logData: any = { type: searchType, domain: cleanDomain }
+    const logData: any = { type: searchType, domain: cleanDomain, terms: parsed.terms.length }
     if (searchQuery && searchQuery.trim()) {
       logData.search = searchQuery.trim()
     }
@@ -41,7 +45,7 @@ export async function POST(request: NextRequest) {
     throwIfAborted(request)
 
     // Call the new data getter function
-    const credentialsData = await getCredentialsDataOptimized(cleanDomain, filters, pagination, searchQuery, searchType, body.keywordMode, signal)
+    const credentialsData = await getCredentialsDataOptimized(parsed, filters, pagination, searchQuery, searchType, body.keywordMode, signal)
 
     // Check abort after operations
     throwIfAborted(request)
@@ -71,7 +75,7 @@ export async function POST(request: NextRequest) {
 }
 
 async function getCredentialsDataOptimized(
-  query: string,
+  parsed: import("@/lib/query-parser").ParsedQuery,
   filters?: any,
   pagination?: any,
   searchQuery?: string,
@@ -99,44 +103,19 @@ async function getCredentialsDataOptimized(
   // ==========================================
   // 1. BUILD PREWHERE (Main Table Filters)
   // ==========================================
-  // PREWHERE is the key to speed in ClickHouse. 
-  // It filters before JOIN and before reading heavy columns.
+  // Use the shared query builder for the main domain/keyword condition
 
   const prewhereConditions: string[] = []
   const params: Record<string, any> = {}
 
   if (searchType === 'domain') {
-    // DOMAIN OPTIMIZATION:
-    // 1. Check Exact Match domain
-    // 2. Check Subdomain using endsWith (much faster than ilike/regex)
-    // 3. Fallback to URL pattern match only if needed
-    
-    params['targetDomain'] = query
-    params['dotTargetDomain'] = '.' + query
-    
-    // Logic: Domain column exact match OR Domain column ends with .target.com
-    // This leverages suffix index if available, or at least fast string scan
-    prewhereConditions.push(`(
-      c.domain = {targetDomain:String} OR 
-      endsWith(c.domain, {dotTargetDomain:String}) OR
-      c.url ilike {urlPattern:String} 
-    )`)
-    // Fallback URL pattern for catch-all
-    params['urlPattern'] = `%${query}%`
-
+    const built = buildDomainReconCondition(parsed, { notNullCheck: false })
+    prewhereConditions.push(`(${built.condition})`)
+    Object.assign(params, built.params)
   } else {
-    // KEYWORD SEARCH
-    params['keyword'] = query
-    params['likeKeyword'] = `%${query}%`
-    
-    if (keywordMode === 'domain-only') {
-      prewhereConditions.push(`(c.domain ilike {likeKeyword:String})`)
-    } else {
-      // Optimization: multiSearchAnyCase is faster than OR OR OR
-      // But for simplicity and param binding, we use ilike in PREWHERE 
-      // because PREWHERE already significantly reduces cost.
-      prewhereConditions.push(`(c.url ilike {likeKeyword:String} OR c.domain ilike {likeKeyword:String})`)
-    }
+    const built = buildKeywordReconCondition(parsed, keywordMode)
+    prewhereConditions.push(`(${built.condition})`)
+    Object.assign(params, built.params)
   }
 
   // Additional Filters to PREWHERE (To filter faster at the start)