From f20d45aa188e88d59fa6bfe75f2a68899700e044 Mon Sep 17 00:00:00 2001 From: 4aiman Date: Fri, 29 May 2026 22:33:12 +0300 Subject: [PATCH] Bulk seed import: INSERT SELECT instead of per-row loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Was: 13000 individual INSERT OR IGNORE in a loop = 2+ minutes. Now: one INSERT ... SELECT FROM seed.products = seconds. Flow: 1. ATTACH seed DB 2. INSERT OR IGNORE ... SELECT (bulk, new products) 3. UPDATE store products with KBJU + micronutrients (user_edited=0 only) 4. UPDATE nutrients_guessed flag 5. INSERT OR REPLACE seed_version into profile 6. COMMIT (all or nothing — crash-safe) 7. DETACH seed_version written INSIDE transaction — if app crashes before COMMIT, next launch re-does the import. No partial state. --- src-tauri/src/lib.rs | 118 +++++++++++++------------------------------ 1 file changed, 35 insertions(+), 83 deletions(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8b44060..9dcfeba 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -139,7 +139,7 @@ fn import_seed_products(app: tauri::AppHandle) -> Result { .map_err(|e| e.to_string())?; // Emit initial progress - let _ = app.emit("seed_progress", serde_json::json!({"current": 0, "total": total})); + // Bulk import — no per-row progress needed (takes <5 seconds) // Ensure all columns exist in main DB (including micronutrients from ION import) let extra_cols = [ @@ -155,93 +155,40 @@ fn import_seed_products(app: tauri::AppHandle) -> Result { let _ = main_db.execute(&format!("ALTER TABLE products ADD COLUMN {}", col), []); } - // Read ALL columns from seed DB including micronutrients - let mut stmt = seed_db.prepare( - "SELECT name, brand, kcal_per_100g, protein_per_100g, fat_per_100g, + // ATTACH seed DB to main DB and do bulk INSERT + UPDATE in SQL (no per-row loop) + let seed_str = seed_path.to_str().unwrap_or_default().to_string(); + main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str]) + .map_err(|e| e.to_string())?; + + let mut inserted = 0usize; + + // Bulk INSERT new products from seed (one SQL statement, not a loop) + main_db.execute("BEGIN", []).map_err(|e| e.to_string())?; + inserted = main_db.execute( + "INSERT OR IGNORE INTO main.products ( + name, brand, kcal_per_100g, protein_per_100g, fat_per_100g, + carbs_per_100g, fiber_per_100g, composition, source, store_id, + name_lower, weight_g, category, + sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg, + iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg, + vit_e_mg, carotene_mcg, nutrients_guessed + ) + SELECT name, brand, kcal_per_100g, protein_per_100g, fat_per_100g, carbs_per_100g, fiber_per_100g, composition, source, store_id, name_lower, weight_g, category, sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg, iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg, vit_e_mg, carotene_mcg, COALESCE(nutrients_guessed, 0) - FROM products" + FROM seed.products", + [], ).map_err(|e| e.to_string())?; - - let mut inserted = 0usize; - let mut processed = 0usize; - - let rows = stmt.query_map([], |row| { - // 26 columns: 0-12 strings/basic, 13-25 micronutrients - Ok(( - row.get::<_, Option>(0)?, // name - row.get::<_, Option>(1)?, // brand - row.get::<_, Option>(2)?, // kcal - row.get::<_, Option>(3)?, // protein - row.get::<_, Option>(4)?, // fat - row.get::<_, Option>(5)?, // carbs - row.get::<_, Option>(6)?, // fiber - row.get::<_, Option>(7)?, // composition - row.get::<_, Option>(8)?, // source - row.get::<_, Option>(9)?, // store_id - row.get::<_, Option>(10)?, // name_lower - row.get::<_, Option>(11)?, // weight_g - row.get::<_, Option>(12)?, // category - row.get::<_, Option>(13)?, // sodium_mg - row.get::<_, Option>(14)?, // potassium_mg - row.get::<_, Option>(15)?, // calcium_mg - row.get::<_, Option>(16)?, // magnesium_mg - row.get::<_, Option>(17)?, // phosphorus_mg - row.get::<_, Option>(18)?, // iron_mg - row.get::<_, Option>(19)?, // vit_a_mcg - row.get::<_, Option>(20)?, // vit_b1_mg - row.get::<_, Option>(21)?, // vit_b2_mg - row.get::<_, Option>(22)?, // vit_b3_mg - row.get::<_, Option>(23)?, // vit_c_mg - row.get::<_, Option>(24)?, // vit_e_mg - row.get::<_, Option>(25)?, // carotene_mcg - row.get::<_, Option>(26)?, // nutrients_guessed - )) - }).map_err(|e| e.to_string())?; - - main_db.execute("BEGIN", []).map_err(|e| e.to_string())?; - for row in rows { - let r = row.map_err(|e| e.to_string())?; - main_db.execute( - "INSERT OR IGNORE INTO products ( - name, brand, kcal_per_100g, protein_per_100g, fat_per_100g, - carbs_per_100g, fiber_per_100g, composition, source, store_id, - name_lower, weight_g, category, - sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg, - iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg, - vit_e_mg, carotene_mcg, nutrients_guessed - ) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27)", - rusqlite::params![ - r.0, r.1, r.2, r.3, r.4, r.5, r.6, r.7, r.8, r.9, - r.10, r.11, r.12, r.13, r.14, r.15, r.16, r.17, r.18, r.19, - r.20, r.21, r.22, r.23, r.24, r.25, r.26 - ], ).map_err(|e| e.to_string())?; inserted += 1; processed += 1; - // Emit progress every 200 rows - if processed % 200 == 0 { - let _ = app.emit("seed_progress", serde_json::json!({"current": processed, "total": total})); - } - } - - main_db.execute( - "INSERT OR REPLACE INTO profile (key, value) VALUES ('seed_version', ?1)", - [seed_version.to_string()], - ).map_err(|e| e.to_string())?; - - main_db.execute("COMMIT", []).map_err(|e| e.to_string())?; - - // Sync enriched data for EXISTING products (INSERT OR IGNORE skips them). - // ATTACH seed DB and run bulk UPDATE to copy nutrients_guessed flag and micronutrients. - let seed_str = seed_path.to_str().unwrap_or_default().to_string(); - if let Ok(_) = main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str]) { - // Sync nutrients_guessed flag - let _ = main_db.execute_batch( + // Update KBJU + micronutrients for existing store products (seed already attached above) + // Sync nutrients_guessed flag + let _ = main_db.execute_batch( "UPDATE main.products SET nutrients_guessed = 1 WHERE (nutrients_guessed IS NULL OR nutrients_guessed = 0) AND name_lower IN (SELECT s.name_lower FROM seed.products s WHERE s.nutrients_guessed = 1)" @@ -272,12 +219,17 @@ fn import_seed_products(app: tauri::AppHandle) -> Result { WHERE (main.products.user_edited IS NULL OR main.products.user_edited = 0) AND main.products.source IN ('magnit', '5ka', 'kdv', 'ion') AND main.products.name_lower IN (SELECT s2.name_lower FROM seed.products s2)" - ); - let _ = main_db.execute("DETACH DATABASE seed", []); - } + ); - // Final progress - let _ = app.emit("seed_progress", serde_json::json!({"current": total, "total": total})); + // Update seed version AFTER all inserts+updates succeed + main_db.execute( + "INSERT OR REPLACE INTO profile (key, value) VALUES ('seed_version', ?1)", + [seed_version.to_string()], + ).map_err(|e| e.to_string())?; + + main_db.execute("COMMIT", []).map_err(|e| e.to_string())?; + + let _ = main_db.execute("DETACH DATABASE seed", []); Ok(format!("imported:{}:{}", inserted, total)) }