Merge pull request #752 from doerfli/feature/mistral-ai-ocr

use mistral ai (#748)

Author: doerfli

Gemfile

@@ -100,3 +100,5 @@ gem "ruby-vips", "~> 2.2"
 gem "nokogiri", "~> 1.19"
 
 gem "propshaft", "~> 1.3"
+
+gem "omniai-mistral", "~> 3.0"

Gemfile.lock

@@ -147,6 +147,7 @@ GEM
       reline (>= 0.3.8)
     debug_inspector (1.2.0)
     diff-lcs (1.6.2)
+    domain_name (0.6.20240107)
     dotenv (3.2.0)
     dotenv-rails (3.2.0)
       dotenv (= 3.2.0)
@@ -177,9 +178,20 @@ GEM
     ffi (1.17.3-x86_64-darwin)
     ffi (1.17.3-x86_64-linux-gnu)
     ffi (1.17.3-x86_64-linux-musl)
+    ffi-compiler (1.3.2)
+      ffi (>= 1.15.5)
+      rake
     globalid (1.3.0)
       activesupport (>= 6.1)
     hashie (5.0.0)
+    http (5.3.1)
+      addressable (~> 2.8)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      llhttp-ffi (~> 0.5.0)
+    http-cookie (1.1.0)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
     i18n (1.14.8)
       concurrent-ruby (~> 1.0)
     image_processing (1.14.0)
@@ -222,6 +234,9 @@ GEM
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    llhttp-ffi (0.5.1)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
     logger (1.7.0)
     loofah (2.25.0)
       crass (~> 1.0.2)
@@ -279,6 +294,16 @@ GEM
       snaky_hash (~> 2.0, >= 2.0.3)
       version_gem (~> 1.1, >= 1.1.9)
     observer (0.1.2)
+    omniai (3.2.0)
+      base64
+      event_stream_parser
+      http
+      logger
+      zeitwerk
+    omniai-mistral (3.0.0)
+      event_stream_parser
+      omniai (~> 3.0)
+      zeitwerk
     omniauth (2.1.4)
       hashie (>= 3.4.6)
       logger
@@ -536,6 +561,7 @@ DEPENDENCIES
   listen (>= 3.0.5)
   mini_magick (~> 5.3)
   nokogiri (~> 1.19)
+  omniai-mistral (~> 3.0)
   omniauth-auth0 (~> 3.1)
   omniauth-rails_csrf_protection (~> 2.0)
   pg (>= 0.18)
@@ -603,6 +629,7 @@ CHECKSUMS
   debug (1.11.1) sha256=2e0b0ac6119f2207a6f8ac7d4a73ca8eb4e440f64da0a3136c30343146e952b6
   debug_inspector (1.2.0) sha256=9bdfa02eebc3da163833e6a89b154084232f5766087e59573b70521c77ea68a2
   diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
+  domain_name (0.6.20240107) sha256=5f693b2215708476517479bf2b3802e49068ad82167bcd2286f899536a17d933
   dotenv (3.2.0) sha256=e375b83121ea7ca4ce20f214740076129ab8514cd81378161f11c03853fe619d
   dotenv-rails (3.2.0) sha256=657e25554ba622ffc95d8c4f1670286510f47f2edda9f68293c3f661b303beab
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
@@ -623,8 +650,12 @@ CHECKSUMS
   ffi (1.17.3-x86_64-darwin) sha256=1f211811eb5cfaa25998322cdd92ab104bfbd26d1c4c08471599c511f2c00bb5
   ffi (1.17.3-x86_64-linux-gnu) sha256=3746b01f677aae7b16dc1acb7cb3cc17b3e35bdae7676a3f568153fb0e2c887f
   ffi (1.17.3-x86_64-linux-musl) sha256=086b221c3a68320b7564066f46fed23449a44f7a1935f1fe5a245bd89d9aea56
+  ffi-compiler (1.3.2) sha256=a94f3d81d12caf5c5d4ecf13980a70d0aeaa72268f3b9cc13358bcc6509184a0
   globalid (1.3.0) sha256=05c639ad6eb4594522a0b07983022f04aa7254626ab69445a0e493aa3786ff11
   hashie (5.0.0) sha256=9d6c4e51f2a36d4616cbc8a322d619a162d8f42815a792596039fc95595603da
+  http (5.3.1) sha256=c50802d8e9be3926cb84ac3b36d1a31fbbac383bc4cbecdce9053cb604231d7d
+  http-cookie (1.1.0) sha256=38a5e60d1527eebc396831b8c4b9455440509881219273a6c99943d29eadbb19
+  http-form_data (2.3.0) sha256=cc4eeb1361d9876821e31d7b1cf0b68f1cf874b201d27903480479d86448a5f3
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   image_processing (1.14.0) sha256=754cc169c9c262980889bec6bfd325ed1dafad34f85242b5a07b60af004742fb
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
@@ -645,6 +676,7 @@ CHECKSUMS
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.9.0) sha256=db9e4424e0e5834480385197c139cb6b0ae0ef28cc13310cfd1ca78377d59c67
+  llhttp-ffi (0.5.1) sha256=9a25a7fc19311f691a78c9c0ac0fbf4675adbd0cca74310228fdf841018fa7bc
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   loofah (2.25.0) sha256=df5ed7ac3bac6a4ec802df3877ee5cc86d027299f8952e6243b3dac446b060e6
   mail (2.9.0) sha256=6fa6673ecd71c60c2d996260f9ee3dd387d4673b8169b502134659ece6d34941
@@ -672,6 +704,8 @@ CHECKSUMS
   nokogiri (1.19.0-x86_64-linux-musl) sha256=1c4ca6b381622420073ce6043443af1d321e8ed93cc18b08e2666e5bd02ffae4
   oauth2 (2.0.18) sha256=bacf11e470dfb963f17348666d0a75c7b29ca65bc48fd47be9057cf91a403287
   observer (0.1.2) sha256=d8a3107131ba661138d748e7be3dbafc0d82e732fffba9fccb3d7829880950ac
+  omniai (3.2.0) sha256=cb052d4027a91d26af4fe5b3f0a9436080b1b1821d5d2a6a43ff0f7f76053358
+  omniai-mistral (3.0.0) sha256=e4958b27d56d1dda82c5ae824422bdee403a7f2d0c2d3b747e4c8e755d007c7b
   omniauth (2.1.4) sha256=42a05b0496f0d22e1dd85d42aaf602f064e36bb47a6826a27ab55e5ba608763c
   omniauth-auth0 (3.1.1) sha256=3d9e83377b37394db077cf27082d29ccff93158f072d92fc59f1e88798c6c2b2
   omniauth-oauth2 (1.9.0) sha256=ed15f6d9d20991807ce114cc5b9c1453bce3645b64e51c68c90cff5ff153fee8

README.md

@@ -7,7 +7,11 @@ Runtime docker images are available.
 ## Features
 
 - Recipe management with images and OCR text recognition
+- **AI-powered recipe extraction with two methods:**
+  - **Mistral + OpenAI**: Image → Mistral OCR (markdown) → OpenAI parsing (structured recipe)
+  - **OpenAI Direct**: Image → OpenAI (structured recipe)
 - AI-powered OCR text cleanup using GPT-4 Mini
+- Multiple recipes detection from single image
 - Tagging and search functionality
 - Auth0 authentication
 - AWS S3 file storage
@@ -38,9 +42,9 @@ AWS_ACCESS_KEY_ID=AAAAAAA
 AWS_SECRET_ACCESS_KEY=BBBBB
 ```
 
-### OpenAI API Key (for OCR text cleanup)
+### OpenAI API Key (for AI recipe extraction and OCR cleanup)
 
-For the GPT-powered OCR text cleanup feature, configure your OpenAI API key:
+For the AI-powered recipe extraction and OCR cleanup features, configure your OpenAI API key:
 
 **Development:**
 ```bash
@@ -64,8 +68,26 @@ dokku config:set your-app-name OPENAI_CLEANUP_PROMPT_DE="Your custom German prom
 **Optional Configuration:**
 - `OPENAI_CLEANUP_PROMPT_EN`: Override the default English cleanup prompt
 - `OPENAI_CLEANUP_PROMPT_DE`: Override the default German cleanup prompt
-- `OPENAI_PROMPT_ID`: Override the default OpenAI prompt ID for OCR (default: `pmpt_69389bf4c7a481909d47bcf85f423781063a569321686620`)
-- `OPENAI_PROMPT_VERSION`: Override the default OpenAI prompt version for OCR (default: `8`)
+- `OPENAI_PROMPT_OCR_ID`: Override the default OpenAI prompt ID for direct OCR (default: `pmpt_694514e453388194a1e4c121407ef02204bec5d20e21b070`)
+- `OPENAI_PROMPT_OCR_VERSION`: Override the default OpenAI prompt version for direct OCR (default: `2`)
+- `OPENAI_MARKDOWN_PROMPT_ID`: Override the default OpenAI prompt ID for markdown parsing (default: `pmpt_696554b87ef88190bbc1156b6c5fe84f0050d5451e60ae6c`)
+- `OPENAI_MARKDOWN_PROMPT_VERSION`: Override the default OpenAI prompt version for markdown parsing (default: `2`)
+
+### Mistral AI API Key (for two-phase OCR)
+
+For the Mistral + OpenAI two-phase recipe extraction, configure your Mistral AI API key:
+
+**Development:**
+```bash
+export MISTRAL_API_KEY=your_mistral_api_key_here
+```
+
+**Production/Dokku:**
+```bash
+dokku config:set your-app-name MISTRAL_API_KEY=your_actual_mistral_api_key_here
+```
+
+**Note:** The Mistral API key is only required if you plan to use the "Mistral + OpenAI" recognition method. The "OpenAI Direct" method only requires the OpenAI API key. 
 
 
 ## Start development server
@@ -80,5 +102,20 @@ yarn build --watch
 
 ## Start production via docker
 
-Use provided `docker-compose.prod.yml` file for startup of postgres db and container for the rails app. Don't forget to set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `OPENAI_API_KEY` and db passwords. To change name of S3 bucket (_reeper_) and region (_eu-central-1_) use `S3_BUCKET_NAME` and `S3_BUCKET_REGION` environment variables. 
+Use provided `docker-compose.prod.yml` file for startup of postgres db and container for the rails app. Don't forget to set the following environment variables:
+
+**Required:**
+- `AWS_ACCESS_KEY_ID`: AWS access key
+- `AWS_SECRET_ACCESS_KEY`: AWS secret key
+- `OPENAI_API_KEY`: OpenAI API key for AI recipe extraction
+- Database passwords
+
+**Optional:**
+- `MISTRAL_API_KEY`: Mistral AI API key (only needed for "Mistral + OpenAI" method)
+- `S3_BUCKET_NAME`: Override S3 bucket name (default: _reeper_)
+- `S3_BUCKET_REGION`: Override S3 region (default: _eu-central-1_)
+- `OPENAI_PROMPT_OCR_ID`: Custom OpenAI prompt ID for direct OCR
+- `OPENAI_PROMPT_OCR_VERSION`: Custom OpenAI prompt version
+- `OPENAI_MARKDOWN_PROMPT_ID`: Custom OpenAI prompt ID for markdown parsing
+- `OPENAI_MARKDOWN_PROMPT_VERSION`: Custom OpenAI prompt version 
 

app/controllers/ocr_controller.rb

@@ -59,12 +59,32 @@ def save_text
   def scan
     # Process only the first uploaded file
     file = params[:files].first
+    ai_method = params[:ai_method] || 'openai_direct'
 
     begin
-      magic_data_json = openai_service.ocr(file.tempfile, file.content_type)
+      # Process based on selected AI method
+      magic_data_json = if ai_method == 'mistral_openai'
+        # Two-phase: Mistral OCR -> OpenAI parsing
+        markdown = mistral_service.ocr_to_markdown(file.tempfile, file.content_type)
+
+        if markdown.blank?
+          raise "Mistral OCR returned empty markdown"
+        end
+
+        openai_service.parse_markdown_to_recipes(markdown)
+      else
+        # Direct OpenAI OCR
+        openai_service.ocr(file.tempfile, file.content_type)
+      end
+
+      if magic_data_json.empty?
+        raise "No recipes extracted from image"
+      end
+
+      logger.debug "OCR extracted recipes: #{magic_data_json}"
 
       # Save full OCR result array to database and store id in flash to avoid flash size limits
-      ocrresult = OcrResult.create(result: magic_data_json.to_json)
+      ocrresult = OcrResult.create(result: magic_data_json.to_json, ai_method: ai_method)
       ocrresult.image.attach(file)
       ocrresult.save
 
@@ -80,7 +100,7 @@ def scan
         render json: { success: true, redirect_url: new_recipe_path }
       end
     rescue JSON::ParserError => e
-      logger.error "OCR JSON parse error: #{e.message}"
+      logger.error "OCR JSON parse error: #{e}"
       render json: { success: false, error: I18n.t('ocr.errors.parse_failed') }
     rescue => e
       logger.error "OCR error: #{e.message}"
@@ -154,6 +174,7 @@ def select_recipe
   def reparse_image
     @recipe = Recipe.find(params[:id])
     attachment_id = params[:attachment_id]
+    ai_method = params[:ai_method] || 'openai_direct'
 
     begin
       # Find the selected image attachment
@@ -164,17 +185,33 @@ def reparse_image
       image_file = blob.download
       content_type = blob.content_type
 
-      # Create a temporary file for the OpenAI service
+      # Create a temporary file for the AI service
       temp_file = Tempfile.new(['recipe_image', File.extname(blob.filename.to_s)])
       temp_file.binmode
       temp_file.write(image_file)
       temp_file.rewind
 
-      # Call OpenAI service to parse the image
-      magic_data_json = openai_service.ocr(temp_file, content_type)
+      # Process based on selected AI method
+      magic_data_json = if ai_method == 'mistral_openai'
+        # Two-phase: Mistral OCR -> OpenAI parsing
+        markdown = mistral_service.ocr_to_markdown(temp_file, content_type)
+
+        if markdown.blank?
+          raise "Mistral OCR returned empty markdown"
+        end
+
+        openai_service.parse_markdown_to_recipes(markdown)
+      else
+        # Direct OpenAI OCR
+        openai_service.ocr(temp_file, content_type)
+      end
+
+      if magic_data_json.empty?
+        raise "No recipes extracted from image"
+      end
 
       # Save full OCR result array to database
-      ocrresult = OcrResult.create(result: magic_data_json.to_json)
+      ocrresult = OcrResult.create(result: magic_data_json.to_json, ai_method: ai_method)
       ocrresult.image.attach(blob)
       ocrresult.save
 
@@ -209,4 +246,8 @@ def reparse_image
   def openai_service
     @openai_service ||= OpenaiService.new
   end
+
+  def mistral_service
+    @mistral_service ||= MistralaiService.new
+  end
 end

app/controllers/ocr_debug_controller.rb

@@ -0,0 +1,33 @@
+class OcrDebugController < ApplicationController
+  include Secured
+
+  def index
+    @page_title = I18n.t('ocr_debug.title')
+  end
+
+  def upload
+    file = params[:files].first
+
+    # Call Mistral AI OCR
+    markdown = mistralai_service.ocr_to_markdown(file.tempfile, file.content_type)
+
+    # Create OcrResult and attach the image
+    ocr_result = OcrResult.create(result: markdown)
+    ocr_result.image.attach(file)
+    ocr_result.save
+
+    # Return JSON with redirect to show page
+    render json: { success: true, redirect_url: ocr_debug_path(ocr_result.id) }
+  end
+
+  def show
+    @ocr_result = OcrResult.find(params[:id])
+    render plain: @ocr_result.result
+  end
+
+  private
+
+  def mistralai_service
+    @mistralai_service ||= MistralaiService.new
+  end
+end

app/javascript/controllers/dropzone_controller.js

@@ -98,6 +98,12 @@ export default class extends Controller {
       formData.append('files[]', files[i])
     }
 
+    // Add AI method selection if present
+    const aiMethodSelect = document.querySelector('select[name="ai_method"]')
+    if (aiMethodSelect) {
+      formData.append('ai_method', aiMethodSelect.value)
+    }
+
     // Upload to server
     fetch(this.urlValue, {
       method: 'POST',

app/javascript/controllers/reparse_controller.js

@@ -1,12 +1,19 @@
 import { Controller } from "@hotwired/stimulus"
 
 export default class extends Controller {
-  static targets = ["button", "buttonText", "spinner"]
+  static targets = ["button", "buttonText", "spinner", "aiMethod", "aiMethodField"]
 
   onSubmit(event) {
     const form = event.target
     const button = form.querySelector('[data-reparse-target="button"]')
 
+    // Get selected AI method and populate hidden field
+    const selectedMethod = document.querySelector('select[name="ai_method"]')
+    const aiMethodField = form.querySelector('[data-reparse-target="aiMethodField"]')
+    if (selectedMethod && aiMethodField) {
+      aiMethodField.value = selectedMethod.value
+    }
+    
     if (button) {
       // Find the button text and spinner within this specific button
       const buttonText = button.querySelector('[data-reparse-target="buttonText"]')

app/services/mistralai_service.rb

@@ -0,0 +1,45 @@
+class MistralaiService
+  def initialize
+    # API key validation happens at runtime (not initialization) to allow
+    # the service to be instantiated even when Mistral AI is not configured.
+    # This is intentional - the key is only checked when the service is actually used.
+    api_key = ENV['MISTRAL_API_KEY']
+    raise "Mistral API key not configured. Set MISTRAL_API_KEY environment variable" if api_key.blank?
+
+    @client = OmniAI::Mistral::Client.new
+  end
+
+  def ocr_to_markdown(image_file, content_type)
+    # Read and encode the image file as base64
+    image_data = if image_file.respond_to?(:read)
+      # Handle uploaded file (Tempfile)
+      image_file.rewind
+      data = Base64.strict_encode64(image_file.read)
+      image_file.rewind  # Reset for potential subsequent reads
+      data
+    elsif image_file.is_a?(String)
+      # Handle file path
+      File.open(image_file, 'rb') { |f| Base64.strict_encode64(f.read) }
+    else
+      raise ArgumentError, "Invalid image_file type"
+    end
+
+    # Determine the image format from content_type or file extension
+    image_format = case content_type
+      when /jpeg|jpg/ then 'jpeg'
+      when /png/ then 'png'
+      when /webp/ then 'webp'
+      when /heic|heif/ then 'heic'
+      else 'jpeg' # default
+    end
+
+    Rails.logger.debug "Image format detected: #{image_format} for content type: #{content_type}"
+    filedata = "data:image/#{image_format};base64,#{image_data}"
+
+    response = @client.ocr(filedata, kind: :image)
+    recognized_markdown = response.pages[0].markdown
+
+    Rails.logger.debug "Mistral OCR result:\n#{recognized_markdown[0..100]}..."
+    recognized_markdown
+  end
+end