add custom extensions + update docs

Abdennacer LACHIHEB · Abdennacer LACHIHEB · commit 03db010a19dc · 2024-09-22T21:32:45.000+02:00
diff --git a/.env.md b/.env.md
@@ -20,6 +20,21 @@ These variables are used for image processing:
 
 ## Examples:
 
+- **GROQ** (Recommended for text processing)
+
+Get your free api key from here: https://console.groq.com/keys
+```bash
+# API and MODEL used for documents processing
+TEXT_API_END_POINT=https://api.groq.com/openai/v1
+TEXT_MODEL_NAME=llama3-70b-8192
+TEXT_API_KEYS=["gsk_xxx","gsk_yyy"]
+
+# API and MODEL used for images processing
+IMAGE_API_END_POINT=https://api.groq.com/openai/v1
+IMAGE_MODEL_NAME=llava-v1.5-7b-4096-preview
+IMAGE_API_KEYS=["gsk_xxx","gsk_yyy"]
+```
+
 - **OPENAI**
 ```bash
 # API and MODEL used for documents processing
@@ -33,19 +48,6 @@ TIMAGE_MODEL_NAME=gpt-4o
 TIMAGE_API_KEYS=["sk-xxx","sk-yyy"]
 ```
 
-- **GROQ**
-```bash
-# API and MODEL used for documents processing
-TEXT_API_END_POINT=https://api.groq.com/openai/v1
-TEXT_MODEL_NAME=llama3-70b-8192
-TEXT_API_KEYS=["gsk_xxx","gsk_yyy"]
-
-# API and MODEL used for images processing ( No vision models for GROQ yet)
-IMAGE_API_END_POINT=http://localhost:11434/v1
-IMAGE_MODEL_NAME=moondream:latest
-IMAGE_API_KEYS=["ollama"]
-```
-
 - **OLLAMA**
 ```bash
 # API and MODEL used for documents processing
@@ -61,6 +63,9 @@ IMAGE_API_KEYS=["ollama"]
 
 
 - **HUGGING FACE**
+
+Get your free api key from here: https://huggingface.co/settings/tokens
+
 ```bash
 # API and MODEL used for documents processing
 TEXT_API_END_POINT=https://api-inference.huggingface.co/v1
diff --git a/.gitignore b/.gitignore
@@ -19,4 +19,4 @@ backend/venv/
 venv/
 backend/env/
 **/__pycache__/
-backend/app/*.db
+backend/**/*.db
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
-# FileWizardAi
+# FileWizardAI
 
 ## Description
 
-FileWizardAi is a Python/Angular project designed to automatically organize your files into a well-structured directory
+FileWizardAI is a Python/Angular project designed to automatically organize your files into a well-structured directory
 hierarchy and rename them according to their content. This tool is ideal for anyone looking to declutter their digital
 workspace by sorting files into appropriate folders and providing descriptive names, making it easier to manage and
 locate files. Additionally, it allows you to input a text prompt and instantly searches for files that are related to
@@ -66,7 +66,7 @@ Make sure you have Python installed on your machine.
 First, clone the repository:
 
 ```bash
-git clone https://github.com/AIxHunter/FileWizardAi.git
+git clone https://github.com/AIxHunter/FileWizardAI.git
 ```
 
 Navigate to the backend folder and update your `.env` file according to the [documentation](.env.md). Then, install the
diff --git a/backend/app/run.py b/backend/app/run.py
@@ -74,6 +74,7 @@ def load_documents(path: str, recursive: bool, required_exts: list):
         input_dir=path,
         recursive=recursive,
         required_exts=required_exts,
+        errors='ignore'
     )
     splitter = TokenTextSplitter(chunk_size=6144)
     documents = []
diff --git a/backend/app/static/index.html b/backend/app/static/index.html
@@ -10,6 +10,6 @@
 <style>html,body{height:100%}body{margin:0;font-family:Roboto,Helvetica Neue,sans-serif}</style><link rel="stylesheet" href="styles.e06a9809e0fa01f5.css" media="print" onload="this.media='all'"><noscript><link rel="stylesheet" href="styles.e06a9809e0fa01f5.css"></noscript></head>
 <body>
   <app-root></app-root>
-<script src="runtime.89bfad0fe920d2c9.js" type="module"></script><script src="polyfills.794d7387aea30963.js" type="module"></script><script src="main.8a6092f762bfc1a2.js" type="module"></script>
+<script src="runtime.89bfad0fe920d2c9.js" type="module"></script><script src="polyfills.794d7387aea30963.js" type="module"></script><script src="main.a1dce65e08b8e85c.js" type="module"></script>
 
 </body></html>
diff --git a/backend/app/static/main.8a6092f762bfc1a2.js b/backend/app/static/main.8a6092f762bfc1a2.js
diff --git a/backend/app/static/main.a1dce65e08b8e85c.js b/backend/app/static/main.a1dce65e08b8e85c.js
diff --git a/backend/requirements.txt b/backend/requirements.txt
@@ -1,5 +1,10 @@
 fastapi
 uvicorn[standard]==0.22.0
-llama-index
 openai
-pydantic-settings
+pydantic-settings
+llama-index
+# needed by llama index
+git+https://github.com/openai/whisper.git
+pydub
+docx2txt
+nbconvert
diff --git a/frontend/src/app/app.component.ts b/frontend/src/app/app.component.ts
@@ -2,20 +2,30 @@ import { Component, HostBinding, Input, QueryList, ViewChildren } from '@angular
 import { DataService } from './data.service';
 import { HttpParams } from "@angular/common/http";
 import { FolderTreeComponent } from './components/folder-tree.component';
+import { NgModel } from '@angular/forms';
 
 @Component({
   selector: 'app-root',
   template: `
       <mat-card class="example-card" appearance="raised">
-          <mat-card-content>
-              <mat-card-header>
-                  <mat-card-title>Update structure</mat-card-title>
-              </mat-card-header>
+
+        <mat-card-header>
+            <mat-card-title>Update structure</mat-card-title>
+        </mat-card-header>
+
+        <mat-card-content>
               <mat-form-field style="margin-right: 1%; width: 33%;" appearance="fill">
                   <mat-label>Root path</mat-label>
                   <input matInput placeholder="Path" [(ngModel)]="rootPath" (ngModelChange)="onPathChange($event)">
               </mat-form-field>
 
+                <mat-form-field appearance="fill">
+                    <mat-label>Add new extension</mat-label>
+                    <input matInput placeholder="Add Extension" [(ngModel)]="newExtension" #extensionControl="ngModel">
+                    <mat-error >Extension should start with a dot ".", ex: '.txt'</mat-error>
+                </mat-form-field>
+                <button style="margin-right: 1%" mat-raised-button (click)="addExtension(extensionControl)">Add</button>
+
               <mat-form-field appearance="fill">
                   <mat-label>Extensions</mat-label>
                   <mat-select [(value)]="filesExts" multiple>
@@ -82,9 +92,10 @@ export class AppComponent {
   successMessage: string = '';
   errorMessage: string = '';
   isLoading: boolean = false;
+  newExtension: string = '';
 
-  filesExts: string[] = ['.pdf', '.txt', '.png', '.jpg', '.jpeg'];
-  filesExtsList: string[] = ['.pdf', '.txt', '.png', '.jpg', '.jpeg'];
+  filesExts: string[] = [".csv", ".docx", ".ipynb", ".jpeg", ".jpg", ".md", ".mp3", ".mp4", ".pdf", ".png", ".ppt"];
+  filesExtsList: string[] = [".csv", ".docx", ".ipynb", ".jpeg", ".jpg", ".md", ".mp3", ".mp4", ".pdf", ".png", ".ppt"];
   @HostBinding('class.active')
   @Input()
   selectedOptions: string[] = [];
@@ -106,7 +117,7 @@ export class AppComponent {
     params = params.set("required_exts", this.filesExts.join(';'))
     this.dataService.getFormattedFiles(params).subscribe((data) => {
       this.original_files = data
-      this.original_files.items = this.original_files.items.map((item: any) => ({src_path: item.src_path.replaceAll("\\\\", "/").replaceAll("\\", "/"), dst_path: item.dst_path}))
+      this.original_files.items = this.original_files.items.map((item: any) => ({ src_path: item.src_path.replaceAll("\\\\", "/").replaceAll("\\", "/"), dst_path: item.dst_path }))
       let res = this.original_files.items.map((item: any) => ({ src_path: `${data.root_path}/${item.src_path}`, dst_path: `${data.root_path}/${item.dst_path}` }))
       this.srcPaths = res.map((r: any) => r.src_path);
       this.dstPaths = res.map((r: any) => r.dst_path);
@@ -136,4 +147,16 @@ export class AppComponent {
     this.childComponents.toArray()[index].highlightFile(matchingFilePath);
   }
 
+  addExtension(extensionControl: NgModel) {
+    if (this.newExtension && !this.filesExtsList.includes(this.newExtension)) {
+      if (this.newExtension.startsWith(".")) {
+        this.filesExtsList.push(this.newExtension);  // Add new option
+        this.filesExts.push(this.newExtension);  // Add new option
+        this.newExtension = '';  // Clear the input field
+      } else {
+        extensionControl.control.setErrors({ customError: true });
+      }
+    }
+  }
+
 }
diff --git a/frontend/src/app/components/search-files.component.ts b/frontend/src/app/components/search-files.component.ts
@@ -13,7 +13,7 @@ import { DataService } from "../data.service";
     <div class="spinner-container">
         <mat-spinner *ngIf="isLoading"></mat-spinner>
     </div>
-    <app-folder-tree title="Searched files" *ngIf="files" [paths]="files" [index]=2></app-folder-tree>
+    <app-folder-tree title="Searched files" *ngIf="files" [paths]="files" [rootPath]="rootPath" [index]=2></app-folder-tree>
   `,
   styles: [`
     .example-full-width {

Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,7 @@ def load_documents(path: str, recursive: bool, required_exts: list):`
`74`	`74`	`input_dir=path,`
`75`	`75`	`recursive=recursive,`
`76`	`76`	`required_exts=required_exts,`
	`77`	`+ errors='ignore'`
`77`	`78`	`)`
`78`	`79`	`splitter = TokenTextSplitter(chunk_size=6144)`
`79`	`80`	`documents = []`