Bulk seed import: INSERT SELECT instead of per-row loop
Was: 13000 individual INSERT OR IGNORE in a loop = 2+ minutes. Now: one INSERT ... SELECT FROM seed.products = seconds. Flow: 1. ATTACH seed DB 2. INSERT OR IGNORE ... SELECT (bulk, new products) 3. UPDATE store products with KBJU + micronutrients (user_edited=0 only) 4. UPDATE nutrients_guessed flag 5. INSERT OR REPLACE seed_version into profile 6. COMMIT (all or nothing — crash-safe) 7. DETACH seed_version written INSIDE transaction — if app crashes before COMMIT, next launch re-does the import. No partial state.
This commit is contained in:
parent
ca45a5acc0
commit
f20d45aa18
1 changed files with 35 additions and 83 deletions
|
|
@ -139,7 +139,7 @@ fn import_seed_products(app: tauri::AppHandle) -> Result<String, String> {
|
|||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// Emit initial progress
|
||||
let _ = app.emit("seed_progress", serde_json::json!({"current": 0, "total": total}));
|
||||
// Bulk import — no per-row progress needed (takes <5 seconds)
|
||||
|
||||
// Ensure all columns exist in main DB (including micronutrients from ION import)
|
||||
let extra_cols = [
|
||||
|
|
@ -155,93 +155,40 @@ fn import_seed_products(app: tauri::AppHandle) -> Result<String, String> {
|
|||
let _ = main_db.execute(&format!("ALTER TABLE products ADD COLUMN {}", col), []);
|
||||
}
|
||||
|
||||
// Read ALL columns from seed DB including micronutrients
|
||||
let mut stmt = seed_db.prepare(
|
||||
"SELECT name, brand, kcal_per_100g, protein_per_100g, fat_per_100g,
|
||||
// ATTACH seed DB to main DB and do bulk INSERT + UPDATE in SQL (no per-row loop)
|
||||
let seed_str = seed_path.to_str().unwrap_or_default().to_string();
|
||||
main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str])
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut inserted = 0usize;
|
||||
|
||||
// Bulk INSERT new products from seed (one SQL statement, not a loop)
|
||||
main_db.execute("BEGIN", []).map_err(|e| e.to_string())?;
|
||||
inserted = main_db.execute(
|
||||
"INSERT OR IGNORE INTO main.products (
|
||||
name, brand, kcal_per_100g, protein_per_100g, fat_per_100g,
|
||||
carbs_per_100g, fiber_per_100g, composition, source, store_id,
|
||||
name_lower, weight_g, category,
|
||||
sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg,
|
||||
iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg,
|
||||
vit_e_mg, carotene_mcg, nutrients_guessed
|
||||
)
|
||||
SELECT name, brand, kcal_per_100g, protein_per_100g, fat_per_100g,
|
||||
carbs_per_100g, fiber_per_100g, composition, source, store_id,
|
||||
name_lower, weight_g, category,
|
||||
sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg,
|
||||
iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg,
|
||||
vit_e_mg, carotene_mcg, COALESCE(nutrients_guessed, 0)
|
||||
FROM products"
|
||||
FROM seed.products",
|
||||
[],
|
||||
).map_err(|e| e.to_string())?;
|
||||
|
||||
let mut inserted = 0usize;
|
||||
let mut processed = 0usize;
|
||||
|
||||
let rows = stmt.query_map([], |row| {
|
||||
// 26 columns: 0-12 strings/basic, 13-25 micronutrients
|
||||
Ok((
|
||||
row.get::<_, Option<String>>(0)?, // name
|
||||
row.get::<_, Option<String>>(1)?, // brand
|
||||
row.get::<_, Option<f64>>(2)?, // kcal
|
||||
row.get::<_, Option<f64>>(3)?, // protein
|
||||
row.get::<_, Option<f64>>(4)?, // fat
|
||||
row.get::<_, Option<f64>>(5)?, // carbs
|
||||
row.get::<_, Option<f64>>(6)?, // fiber
|
||||
row.get::<_, Option<String>>(7)?, // composition
|
||||
row.get::<_, Option<String>>(8)?, // source
|
||||
row.get::<_, Option<String>>(9)?, // store_id
|
||||
row.get::<_, Option<String>>(10)?, // name_lower
|
||||
row.get::<_, Option<f64>>(11)?, // weight_g
|
||||
row.get::<_, Option<String>>(12)?, // category
|
||||
row.get::<_, Option<f64>>(13)?, // sodium_mg
|
||||
row.get::<_, Option<f64>>(14)?, // potassium_mg
|
||||
row.get::<_, Option<f64>>(15)?, // calcium_mg
|
||||
row.get::<_, Option<f64>>(16)?, // magnesium_mg
|
||||
row.get::<_, Option<f64>>(17)?, // phosphorus_mg
|
||||
row.get::<_, Option<f64>>(18)?, // iron_mg
|
||||
row.get::<_, Option<f64>>(19)?, // vit_a_mcg
|
||||
row.get::<_, Option<f64>>(20)?, // vit_b1_mg
|
||||
row.get::<_, Option<f64>>(21)?, // vit_b2_mg
|
||||
row.get::<_, Option<f64>>(22)?, // vit_b3_mg
|
||||
row.get::<_, Option<f64>>(23)?, // vit_c_mg
|
||||
row.get::<_, Option<f64>>(24)?, // vit_e_mg
|
||||
row.get::<_, Option<f64>>(25)?, // carotene_mcg
|
||||
row.get::<_, Option<i64>>(26)?, // nutrients_guessed
|
||||
))
|
||||
}).map_err(|e| e.to_string())?;
|
||||
|
||||
main_db.execute("BEGIN", []).map_err(|e| e.to_string())?;
|
||||
for row in rows {
|
||||
let r = row.map_err(|e| e.to_string())?;
|
||||
main_db.execute(
|
||||
"INSERT OR IGNORE INTO products (
|
||||
name, brand, kcal_per_100g, protein_per_100g, fat_per_100g,
|
||||
carbs_per_100g, fiber_per_100g, composition, source, store_id,
|
||||
name_lower, weight_g, category,
|
||||
sodium_mg, potassium_mg, calcium_mg, magnesium_mg, phosphorus_mg,
|
||||
iron_mg, vit_a_mcg, vit_b1_mg, vit_b2_mg, vit_b3_mg, vit_c_mg,
|
||||
vit_e_mg, carotene_mcg, nutrients_guessed
|
||||
) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27)",
|
||||
rusqlite::params![
|
||||
r.0, r.1, r.2, r.3, r.4, r.5, r.6, r.7, r.8, r.9,
|
||||
r.10, r.11, r.12, r.13, r.14, r.15, r.16, r.17, r.18, r.19,
|
||||
r.20, r.21, r.22, r.23, r.24, r.25, r.26
|
||||
],
|
||||
).map_err(|e| e.to_string())?;
|
||||
inserted += 1;
|
||||
processed += 1;
|
||||
|
||||
// Emit progress every 200 rows
|
||||
if processed % 200 == 0 {
|
||||
let _ = app.emit("seed_progress", serde_json::json!({"current": processed, "total": total}));
|
||||
}
|
||||
}
|
||||
|
||||
main_db.execute(
|
||||
"INSERT OR REPLACE INTO profile (key, value) VALUES ('seed_version', ?1)",
|
||||
[seed_version.to_string()],
|
||||
).map_err(|e| e.to_string())?;
|
||||
|
||||
main_db.execute("COMMIT", []).map_err(|e| e.to_string())?;
|
||||
|
||||
// Sync enriched data for EXISTING products (INSERT OR IGNORE skips them).
|
||||
// ATTACH seed DB and run bulk UPDATE to copy nutrients_guessed flag and micronutrients.
|
||||
let seed_str = seed_path.to_str().unwrap_or_default().to_string();
|
||||
if let Ok(_) = main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str]) {
|
||||
// Sync nutrients_guessed flag
|
||||
let _ = main_db.execute_batch(
|
||||
// Update KBJU + micronutrients for existing store products (seed already attached above)
|
||||
// Sync nutrients_guessed flag
|
||||
let _ = main_db.execute_batch(
|
||||
"UPDATE main.products SET nutrients_guessed = 1
|
||||
WHERE (nutrients_guessed IS NULL OR nutrients_guessed = 0)
|
||||
AND name_lower IN (SELECT s.name_lower FROM seed.products s WHERE s.nutrients_guessed = 1)"
|
||||
|
|
@ -272,12 +219,17 @@ fn import_seed_products(app: tauri::AppHandle) -> Result<String, String> {
|
|||
WHERE (main.products.user_edited IS NULL OR main.products.user_edited = 0)
|
||||
AND main.products.source IN ('magnit', '5ka', 'kdv', 'ion')
|
||||
AND main.products.name_lower IN (SELECT s2.name_lower FROM seed.products s2)"
|
||||
);
|
||||
let _ = main_db.execute("DETACH DATABASE seed", []);
|
||||
}
|
||||
);
|
||||
|
||||
// Final progress
|
||||
let _ = app.emit("seed_progress", serde_json::json!({"current": total, "total": total}));
|
||||
// Update seed version AFTER all inserts+updates succeed
|
||||
main_db.execute(
|
||||
"INSERT OR REPLACE INTO profile (key, value) VALUES ('seed_version', ?1)",
|
||||
[seed_version.to_string()],
|
||||
).map_err(|e| e.to_string())?;
|
||||
|
||||
main_db.execute("COMMIT", []).map_err(|e| e.to_string())?;
|
||||
|
||||
let _ = main_db.execute("DETACH DATABASE seed", []);
|
||||
|
||||
Ok(format!("imported:{}:{}", inserted, total))
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue