find_duplicates.py

# import os

# file1 = r"E:\DSs\MLP\code\TRDG\data\text\arabic_w3m.txt"
# file2 = r"E:\DSs\MLP\code\TRDG\data\text\arabic_kurdish_words.txt"
# # file1 = r"E:\DSs\MLP\code\TRDG\data\text\kurdish_words_only03.txt"
# # file2 = r"E:\DSs\MLP\code\TRDG\data\text\arabic_w3m.txt"
# out_file = r"E:\DSs\MLP\code\TRDG\data\text\350k\arabic_words.txt"

# def find_common_words():
#     if not os.path.exists(file1):
#         print(f"Error: {file1} not found.")
#         return
#     if not os.path.exists(file2):
#         print(f"Error: {file2} not found.")
#         return
        
#     print("Reading first file...")
#     with open(file1, 'r', encoding='utf-8') as f:
#         words1 = set(line.strip() for line in f if line.strip())
        
#     print(f"Loaded {len(words1)} unique words from first file.")
        
#     print("Reading second file...")
#     with open(file2, 'r', encoding='utf-8') as f:
#         words2 = set(line.strip() for line in f if line.strip())
        
#     print(f"Loaded {len(words2)} unique words from second file.")
        
#     # common_words = words1.intersection(words2)
#     common_words = words1.symmetric_difference(words2)
#     print(f"Found {len(common_words)} common words in both files.")
    
#     # Ensure output directory exists
#     os.makedirs(os.path.dirname(out_file), exist_ok=True)
    
#     print(f"Writing common words to {out_file}...")
#     with open(out_file, 'w', encoding='utf-8') as f:
#         for word in sorted(common_words):
#             f.write(word + '\n')
            
#     print("Done!")

# if __name__ == "__main__":
#     find_common_words()


import os
import random
import subprocess
import math
import shutil
import sys
import glob

try:
    from fontTools.ttLib import TTFont
except ImportError:
    print("❌ تکایە سەرەتا fonttools دابەزێنە بەم فەرمانە: pip install fonttools")
    sys.exit(1)

# ==========================================
# ڕێکخستنە سەرەکییەکان
# ==========================================

BASE_DIR = r'E:\DSs\MLP\code\TRDG'
OUTPUT_DIR = os.path.join(BASE_DIR, 'data', 'text','350k', 'imgs','kuar_350')
TEXT_FILE = r"E:\DSs\MLP\code\TRDG\data\text\350k\arabic_kurdish_words.txt"
FONT_DIR = os.path.join(BASE_DIR, '01')
TRDG_SCRIPT = os.path.join(BASE_DIR, 'trdg', 'run.py')
PYTHON_EXEC = sys.executable  

TOTAL_IMAGES_TARGET = 50000 
FONT_SIZE_MIN = 26
FONT_SIZE_MAX = 64

# === بەشی نوێ: ڕێژەی شێواندن ===
# 0.10 واتە 10% ی وێنەکان باکگراوند و کوالێتییان تێکدەدرێت
DISTORTION_PERCENTAGE = 0.25  

def load_and_shuffle_words(file_path):
    print(f"⏳ خوێندنەوەی وشەکان لە {file_path}...")
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            words = [line.strip() for line in f if line.strip()]
        print(f"✅ {len(words)} وشە بارکرا.")
        random.shuffle(words)
        return list(enumerate(words))
    except Exception as e:
        print(f"❌ نەتوانرا فایلەکە بخوێنرێتەوە: {e}")
        return []

def get_font_supported_chars(font_path):
    try:
        font = TTFont(font_path)
        supported_chars = set()
        for table in font['cmap'].tables:
            supported_chars.update(table.cmap.keys())
        return supported_chars
    except:
        return set()

def get_fonts(font_folder):
    if not os.path.exists(font_folder):
        print("❌ فۆڵدەری فۆنتەکان نەدۆزرایەوە.")
        return []
    fonts = [os.path.join(font_folder, f) for f in os.listdir(font_folder) if f.endswith(('.ttf', '.otf'))]
    fonts.sort()
    print(f"✅ {len(fonts)} فۆنت دۆزرایەوە.")
    return fonts

def generate_unique_dataset():
    if not os.path.exists(TRDG_SCRIPT):
        print(f"❌ فایلی run.py نەدۆزرایەوە.")
        return

    if not os.path.exists(OUTPUT_DIR):
        os.makedirs(OUTPUT_DIR)
    
    word_queue = load_and_shuffle_words(TEXT_FILE)
    fonts = get_fonts(FONT_DIR)
    
    if not word_queue or not fonts:
        return

    images_per_font = math.ceil(TOTAL_IMAGES_TARGET / len(fonts))

    print("="*50)
    print(f"📊 پلانی دروستکردن:")
    print(f"   - ئامانج: {TOTAL_IMAGES_TARGET} وێنە")
    print(f"   - ڕێژەی شێواندن (Distortion): {DISTORTION_PERCENTAGE * 100}%")
    print("="*50)

    total_clean = 0
    total_distorted = 0
    
    env = os.environ.copy()
    env['PYTHONPATH'] = BASE_DIR + os.pathsep + env.get('PYTHONPATH', '')

    for font_idx, font_path in enumerate(fonts):
        font_name = os.path.basename(font_path)
        print(f"\n🔠 کارکردن لەسەر فۆنتی: {font_idx} - {font_name}")
        
        supported_chars = get_font_supported_chars(font_path)
        if not supported_chars: continue

        valid_font_words = []
        while len(valid_font_words) < images_per_font and word_queue:
            orig_idx, word = word_queue.pop()
            is_supported = True
            for char in word:
                if ord(char) not in supported_chars and ord(char) not in [0x200C, 0x0020]:
                    is_supported = False
                    break
            if is_supported:
                valid_font_words.append((orig_idx, word))
            
        if not valid_font_words: continue

        size_groups = {}
        for item in valid_font_words:
            random_size = random.randint(FONT_SIZE_MIN, FONT_SIZE_MAX)
            if random_size not in size_groups:
                size_groups[random_size] = []
            size_groups[random_size].append(item)

        for size, batch_items in size_groups.items():
            batch_indices = [idx for idx, w in batch_items]
            batch_words = [w for idx, w in batch_items]
            
            temp_batch_dir = os.path.join(BASE_DIR, "temp_images_processing")
            if os.path.exists(temp_batch_dir): shutil.rmtree(temp_batch_dir)
            os.makedirs(temp_batch_dir)

            temp_text_file = os.path.join(BASE_DIR, "temp_batch_words.txt")
            with open(temp_text_file, 'w', encoding='utf-8') as f:
                f.write('\n'.join(batch_words))

            current_margin = random.randint(4, 10)
            image_height = size + (current_margin * 2)

            # === بڕیاردان بۆ شێواندنی ئەم وەجبەیە ===
            is_distorted = random.random() < DISTORTION_PERCENTAGE
            
            bg_type = "1" # باکگراوندی سپی ئاسایی
            extra_args = []

            if is_distorted:
                # 0: Gaussian Noise (ژاوەژاو) | 2: Quasicrystal (باکگراوندی شێواو)
                bg_type = random.choice(["0", "2"])
                
                # جۆری شێواندن (ڵێڵکردن، شەپۆل، یان هەردووکی)
                distortion_mode = random.choice(["blur"]) #, "wave", "both"
                
                if distortion_mode in ["blur", "both"]:
                    extra_args.append("-rbl") # ڵێڵکردنی ڕاندۆم
                    
                if distortion_mode in ["wave"]:#, "both"
                    # 1: Sine, 2: Cosine, 3: Random
                    extra_args.extend(["-d", str(random.randint(1, 3))]) 

            cmd = [
                PYTHON_EXEC, TRDG_SCRIPT,
                "-c", str(len(batch_words)),
                "-i", temp_text_file,
                "--output_dir", temp_batch_dir,
                "-ft", font_path,
                "-f", str(image_height),
                "-b", bg_type, # دیاریکردنی جۆری باکگراوند
                "-tc", "#000000",
                "-m", str(current_margin),
                "-w", "1",
                "-na", "2" 
            ] + extra_args # زیادکردنی فەرمانەکانی شێواندن ئەگەر هەبوو

            try:
                subprocess.run(cmd, env=env, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
                generated_files = glob.glob(os.path.join(temp_batch_dir, "*.jpg"))
                
                def get_file_number(filename):
                    try: return int(os.path.splitext(filename)[0].split('_')[-1])
                    except: return -1

                generated_files.sort(key=get_file_number)
                files_moved = 0
                
                for i, file_path in enumerate(generated_files):
                    if i < len(batch_indices):
                        original_word_index = batch_indices[i]
                        # پێشگرێک زیاد دەکەین (D_) ئەگەر وێنەکە شێوێندرابێت بۆ ئەوەی دواتر بیانسازیتەوە
                        status_prefix = "D" if is_distorted else "C" 
                        new_filename = f"{status_prefix}_F{font_idx}_S{size}_W{original_word_index}.jpg"
                        dest_path = os.path.join(OUTPUT_DIR, new_filename)
                        
                        shutil.move(file_path, dest_path)
                        files_moved += 1

                if is_distorted:
                    total_distorted += files_moved
                else:
                    total_clean += files_moved
                    
                print(f"   -> قەبارە: {size} | +{files_moved} وێنە ({'شێواو ⚠️' if is_distorted else 'پاک ✅'})", end='\r')
            
            except subprocess.CalledProcessError as e:
                print(f"\n❌ هەڵە لە قەبارەی {size}!")
                print(f"   🛑 Error: {e.stderr.strip() if e.stderr else 'Unknown'}")

            if os.path.exists(temp_text_file):
                try: os.remove(temp_text_file)
                except: pass
            if os.path.exists(temp_batch_dir):
                try: shutil.rmtree(temp_batch_dir)
                except: pass

    print(f"\n\n🎉 پرۆسەکە بە سەرکەوتوویی تەواو بوو.")
    print(f"   ✅ کۆی گشتی وێنەی پاک (Clean): {total_clean}")
    print(f"   ⚠️ کۆی گشتی وێنەی شێوێنراو (Distorted): {total_distorted}")
    print(f"   📊 کۆی گشتی هەمووی: {total_clean + total_distorted}")

if __name__ == "__main__":
    generate_unique_dataset()