4aiman/baldav
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Dedup products, fix labels, fix track comparison, seed v9

Browse source 
Dedup products in Rust import (not JS — JS was too slow/hung):
- Key: (name_lower, source, COALESCE(store_id,''))
- Same product from same store = duplicate, deleted
- Different sources (magnit vs ion vs manual) = separate products
- Reassigns food_log + recipe_ingredients refs before deleting
- UNIQUE INDEX prevents future dupes (INSERT OR IGNORE now works)
- Tested: 16675 -> 8330 in 0.15s on device DB copy

Removed JS dedup migration (replaced by Rust).

Order in import_seed_products:
1. Index name_lower
2. Dedup (reassign refs + delete)
3. UNIQUE INDEX
4. INSERT OR IGNORE from seed
5. UPDATE store products
6. seed_version + COMMIT
This commit is contained in:
4aiman 2026-06-01 14:51:38 +03:00
commit b2b6b6a7ef
2 changed files with 47 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

19
frontend/js/db.js
View file

@ -1153,24 +1153,7 @@ async function runMigrations() {
await addSuppCol("omega6_mg", "REAL"); // омега-6
await addSuppCol("omega9_mg", "REAL"); // омега-9
// Deduplicate products: keep lowest id per name_lower, reassign refs, delete dupes
// All in bulk SQL — no per-row loop
try {
const dupeCount = await select("SELECT COUNT(*) as cnt FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower)");
if (dupeCount[0].cnt > 0) {
// Reassign food_log to kept (MIN) id
await execute(`UPDATE food_log SET product_id = (
SELECT MIN(p2.id) FROM products p2 WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = food_log.product_id)
) WHERE product_id IN (SELECT id FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower))`);
// Reassign recipe_ingredients
await execute(`UPDATE recipe_ingredients SET product_id = (
SELECT MIN(p2.id) FROM products p2 WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = recipe_ingredients.product_id)
) WHERE product_id IN (SELECT id FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower))`);
// Delete dupes
await execute("DELETE FROM products WHERE name_lower IS NOT NULL AND id NOT IN (SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL GROUP BY name_lower)");
console.log(`[db.js] Deduplicated ${dupeCount[0].cnt} products`);
}
} catch (e) { console.error("[db.js] dedup error:", e); }
// Product dedup handled in Rust import_seed_products (UNIQUE INDEX on name_lower+source+store_id)
console.log("[db.js] recipe migrations done, checking seq repair...");

50
src-tauri/src/lib.rs
View file

@ -152,8 +152,51 @@ fn import_seed_products(app: tauri::AppHandle) -> Result<String, String> {
let seed_str = seed_path.to_str().unwrap_or_default().to_string();
main_db.execute("ATTACH DATABASE ?1 AS seed", rusqlite::params![seed_str]).map_err(|e| e.to_string())?;
// Bulk INSERT new products from seed (one SQL statement, not a loop)
main_db.execute("BEGIN", []).map_err(|e| e.to_string())?;
// Create index for fast lookups during dedup and insert
let _ = main_db.execute_batch(
"CREATE INDEX IF NOT EXISTS idx_products_name_lower ON main.products(name_lower);"
);
// Dedup existing products by (name_lower, source, store_id).
// Reassign food_log and recipe_ingredients refs to kept (MIN id) product, then delete dupes.
let _ = main_db.execute_batch(
"UPDATE food_log SET product_id = (
SELECT MIN(p2.id) FROM products p2
WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = food_log.product_id)
AND p2.source = (SELECT p3.source FROM products p3 WHERE p3.id = food_log.product_id)
AND COALESCE(p2.store_id,'') = COALESCE((SELECT p3.store_id FROM products p3 WHERE p3.id = food_log.product_id),'')
) WHERE product_id NOT IN (
SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL
GROUP BY name_lower, source, COALESCE(store_id,'')
);
UPDATE recipe_ingredients SET product_id = (
SELECT MIN(p2.id) FROM products p2
WHERE p2.name_lower = (SELECT p3.name_lower FROM products p3 WHERE p3.id = recipe_ingredients.product_id)
AND p2.source = (SELECT p3.source FROM products p3 WHERE p3.id = recipe_ingredients.product_id)
AND COALESCE(p2.store_id,'') = COALESCE((SELECT p3.store_id FROM products p3 WHERE p3.id = recipe_ingredients.product_id),'')
) WHERE product_id NOT IN (
SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL
GROUP BY name_lower, source, COALESCE(store_id,'')
);
DELETE FROM products WHERE name_lower IS NOT NULL
AND id NOT IN (
SELECT MIN(id) FROM products WHERE name_lower IS NOT NULL
GROUP BY name_lower, source, COALESCE(store_id,'')
);"
);
// Create UNIQUE index to prevent future duplicates
let _ = main_db.execute_batch(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_products_dedup
ON main.products(name_lower, source, COALESCE(store_id, ''))
WHERE name_lower IS NOT NULL;"
);
// Bulk INSERT new products from seed (INSERT OR IGNORE now works with unique index)
let inserted = main_db.execute(
"INSERT OR IGNORE INTO main.products (
name, brand, kcal_per_100g, protein_per_100g, fat_per_100g,
@ -173,10 +216,9 @@ fn import_seed_products(app: tauri::AppHandle) -> Result<String, String> {
[]
).map_err(|e| e.to_string())?;
// Create indexes for fast name_lower lookups (critical: without this UPDATE takes 60s+)
// Seed index for fast UPDATE lookups
let _ = main_db.execute_batch(
"CREATE INDEX IF NOT EXISTS idx_products_name_lower ON main.products(name_lower);
CREATE INDEX IF NOT EXISTS idx_seed_name_lower ON seed.products(name_lower);"
"CREATE INDEX IF NOT EXISTS idx_seed_name_lower ON seed.products(name_lower);"
);
// Update KBJU + micronutrients for existing store products (seed already attached above)