From b2b6b6a7ef6a1b668821e784a4029716075f68c6 Mon Sep 17 00:00:00 2001 From: 4aiman Date: Mon, 1 Jun 2026 14:51:38 +0300 Subject: [PATCH] Dedup products, fix labels, fix track comparison, seed v9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dedup products in Rust import (not JS — JS was too slow/hung): - Key: (name_lower, source, COALESCE(store_id,'')) - Same product from same store = duplicate, deleted - Different sources (magnit vs ion vs manual) = separate products - Reassigns food_log + recipe_ingredients refs before deleting - UNIQUE INDEX prevents future dupes (INSERT OR IGNORE now works) - Tested: 16675 -> 8330 in 0.15s on device DB copy Removed JS dedup migration (replaced by Rust). Order in import_seed_products: 1. Index name_lower 2. Dedup (reassign refs + delete) 3. UNIQUE INDEX 4. INSERT OR IGNORE from seed 5. UPDATE store products 6. seed_version + COMMIT --- frontend/js/db.js | 19 +---------------- src-tauri/src/lib.rs | 50 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/frontend/js/db.js b/frontend/js/db.js index 95140a0..a90f4fd 100644 --- a/frontend/js/db.js +++ b/frontend/js/db.js @@ -1153,24 +1153,7 @@ async function runMigrations() { await addSuppCol("omega6_mg", "REAL"); // омега-6 await addSuppCol("omega9_mg", "REAL"); // омега-9 - // Deduplicate products: keep lowest id per name_lower, reassign refs, delete dupes - // All in bulk SQL — no per-row loop - try { - const dupeCount = await select("SELECT COUNT(*) as cnt FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower)"); - if (dupeCount[0].cnt > 0) { - // Reassign food_log to kept (MIN) id - await execute(`UPDATE food_log SET product_id = ( - SELECT MIN(p2.id) FROM products p2 WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = food_log.product_id) - ) WHERE product_id IN (SELECT id FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower))`); - // Reassign recipe_ingredients - await execute(`UPDATE recipe_ingredients SET product_id = ( - SELECT MIN(p2.id) FROM products p2 WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = recipe_ingredients.product_id) - ) WHERE product_id IN (SELECT id FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower))`); - // Delete dupes - await execute("DELETE FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower)"); - console.log(`[db.js] Deduplicated ${dupeCount[0].cnt} products`); - } - } catch (e) { console.error("[db.js] dedup error:", e); } + // Product dedup handled in Rust import_seed_products (UNIQUE INDEX on name_lower+source+store_id) console.log("[db.js] recipe migrations done, checking seq repair..."); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index dc2bba9..d8e10d6 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -152,8 +152,51 @@ fn import_seed_products(app: tauri::AppHandle) -> Result { let seed_str = seed_path.to_str().unwrap_or_default().to_string(); main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str]).map_err(|e| e.to_string())?; - // Bulk INSERT new products from seed (one SQL statement, not a loop) main_db.execute("BEGIN", []).map_err(|e| e.to_string())?; + + // Create index for fast lookups during dedup and insert + let _ = main_db.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_products_name_lower ON main.products(name_lower);" + ); + + // Dedup existing products by (name_lower, source, store_id). + // Reassign food_log and recipe_ingredients refs to kept (MIN id) product, then delete dupes. + let _ = main_db.execute_batch( + "UPDATE food_log SET product_id = ( + SELECT MIN(p2.id) FROM products p2 + WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = food_log.product_id) + AND p2.source = (SELECT p3.source FROM products p3 WHERE p3.id = food_log.product_id) + AND COALESCE(p2.store_id,'') = COALESCE((SELECT p3.store_id FROM products p3 WHERE p3.id = food_log.product_id),'') + ) WHERE product_id NOT IN ( + SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL + GROUP BY name_lower, source, COALESCE(store_id,'') + ); + + UPDATE recipe_ingredients SET product_id = ( + SELECT MIN(p2.id) FROM products p2 + WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = recipe_ingredients.product_id) + AND p2.source = (SELECT p3.source FROM products p3 WHERE p3.id = recipe_ingredients.product_id) + AND COALESCE(p2.store_id,'') = COALESCE((SELECT p3.store_id FROM products p3 WHERE p3.id = recipe_ingredients.product_id),'') + ) WHERE product_id NOT IN ( + SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL + GROUP BY name_lower, source, COALESCE(store_id,'') + ); + + DELETE FROM products WHERE name_lower IS NOT NULL + AND id NOT IN ( + SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL + GROUP BY name_lower, source, COALESCE(store_id,'') + );" + ); + + // Create UNIQUE index to prevent future duplicates + let _ = main_db.execute_batch( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_products_dedup + ON main.products(name_lower, source, COALESCE(store_id, '')) + WHERE name_lower IS NOT NULL;" + ); + + // Bulk INSERT new products from seed (INSERT OR IGNORE now works with unique index) let inserted = main_db.execute( "INSERT OR IGNORE INTO main.products ( name, brand, kcal_per_100g, protein_per_100g, fat_per_100g, @@ -173,10 +216,9 @@ fn import_seed_products(app: tauri::AppHandle) -> Result { [] ).map_err(|e| e.to_string())?; - // Create indexes for fast name_lower lookups (critical: without this UPDATE takes 60s+) + // Seed index for fast UPDATE lookups let _ = main_db.execute_batch( - "CREATE INDEX IF NOT EXISTS idx_products_name_lower ON main.products(name_lower); - CREATE INDEX IF NOT EXISTS idx_seed_name_lower ON seed.products(name_lower);" + "CREATE INDEX IF NOT EXISTS idx_seed_name_lower ON seed.products(name_lower);" ); // Update KBJU + micronutrients for existing store products (seed already attached above)