From 1f5b393675b4c974b9f8e35b2b6efaab9710fc2a Mon Sep 17 00:00:00 2001 From: aaverbitskiy Date: Wed, 5 Aug 2026 19:25:40 +0000 Subject: [PATCH] bookings/map: read pf_board and board_info with FINAL to dedup ReplacingMergeTree Both tables are ReplacingMergeTree; dedup happens only during background part merges, not at read time. The ongoing etl_rw feed keeps inserting into pf_board, so right after an insert that updates an existing row there is a short window (before the forced merge, min_age_to_force_merge_seconds=15) where the same task_id / board_key exists in two parts. A plain SELECT reads both parts and returns the row twice. The /api/boards query already collapses duplicates via argMax + GROUP BY, but /api/bookings and the two /api/map queries read raw rows, so they could intermittently double a booking (timeline/balloon) or a surface marker. Add FINAL to those three reads (pf_board in bookings + map current, board_info in map surfaces). At 463/515 rows the cost is negligible and it removes the class of transient duplicates regardless of merge timing. Output verified unchanged against the single-part baseline. Co-Authored-By: Claude Opus 4.8 --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index a5ae7d6..36a506b 100644 --- a/main.py +++ b/main.py @@ -221,7 +221,7 @@ async def bookings(city: str = "", dimension: str = "", brand: str = "", manager toString(end_date) AS end_date, brand, company_name - FROM default.pf_board + FROM default.pf_board FINAL WHERE {where} ORDER BY board_id, start_date """ @@ -255,7 +255,7 @@ async def map_data(city: str = "", dimension: str = "", brand: str = "", manager side, latitude AS lat, longitude AS lon - FROM default.board_info + FROM default.board_info FINAL WHERE {where} ORDER BY city, address """ @@ -271,7 +271,7 @@ async def map_data(city: str = "", dimension: str = "", brand: str = "", manager toString(end_date) AS end_date, brand, company_name - FROM default.pf_board + FROM default.pf_board FINAL WHERE board_key != 0 AND pf_board.start_date <= today() AND pf_board.end_date >= today()