#!/usr/bin/env python3 """ FastAPI app with concurrent PDF URL downloads (threadpool). Save as: app_concurrent_downloads.py Run: uvicorn app_concurrent_downloads:app --host 0.0.0.0 --port 8000 Behavior: - Accepts JSON POST to /match_json or multipart to /match (payload_json + upload_files). - 'files' entries may be local paths or HTTP/HTTPS URLs. - All URL downloads are performed concurrently using ThreadPoolExecutor. - Failed downloads are reported in results; processing continues for successful files. - Matching logic (strict all-present -> ACCEPTED / otherwise REJECTED) is unchanged. """ from fastapi import FastAPI, UploadFile, File, Form, HTTPException from fastapi.responses import JSONResponse from pydantic import BaseModel from typing import List, Optional, Dict from pathlib import Path import shutil import re import json from difflib import SequenceMatcher import requests from urllib.parse import urlparse import time import concurrent.futures # optional: rapidfuzz if available try: from rapidfuzz import fuzz HAVE_RAPIDFUZZ = True except Exception: HAVE_RAPIDFUZZ = False import pdfplumber app = FastAPI(title="Land-Doc Table Matcher (Concurrent Downloads)") # ---------------- default FILES (edit if you want) ---------------- DEFAULT_FILES = [ "/mnt/data/371(Plot No)_Suahanta Mondal.pdf", "/mnt/data/348__South Gobindapur__SAINTHIA.pdf", "/mnt/data/345__South Gobindapur__SAINTHIA.pdf", "/mnt/data/1764746873181-519NetureeSAINTHIA.pdf" ] # ---------------- Matching config (tweak if needed) ---------------- REQUIRE_KHATIAN_FOR_FARMER_MATCH = True FARMER_FUZZY_THRESHOLD = 85.0 KHATIAN_CONFIDENT_MIN_SCORE = 60.0 # ---------------- Download config ---------------- DOWNLOAD_DIR = Path("./downloads"); DOWNLOAD_DIR.mkdir(exist_ok=True) UPLOAD_DIR = Path("./uploads"); UPLOAD_DIR.mkdir(exist_ok=True) DOWNLOAD_TIMEOUT = 20 # seconds per request DOWNLOAD_WORKERS = 8 # ThreadPool max workers (tweak for your environment) DOWNLOAD_RETRIES = 1 # simple retry count # ---------------- Pydantic models ---------------- class MatchPayload(BaseModel): daag: List[str] khatian: List[str] farmer: List[str] files: Optional[List[str]] = None # ---------------- Helper functions for URL detection + download ---------------- def is_url(s: str) -> bool: try: p = urlparse(s) return p.scheme in ("http", "https") except Exception: return False def download_pdf_once(url: str, dest_dir: Path, timeout: int = DOWNLOAD_TIMEOUT) -> Optional[str]: """ Download a single PDF URL to dest_dir, return local path on success, None on failure. This does not perform retries — wrapper will handle retries. """ try: r = requests.get(url, stream=True, timeout=timeout) r.raise_for_status() parsed = urlparse(url) fname = Path(parsed.path).name if not fname or not fname.lower().endswith(".pdf"): fname = f"download_{int(time.time()*1000)}.pdf" dest = dest_dir / fname # avoid overwrite if dest.exists(): base = dest.stem suf = 1 while (dest_dir / f"{base}_{suf}.pdf").exists(): suf += 1 dest = dest_dir / f"{base}_{suf}.pdf" with dest.open("wb") as f: for chunk in r.iter_content(chunk_size=8192): if chunk: f.write(chunk) return str(dest.resolve()) except Exception: return None def download_with_retries(url: str, dest_dir: Path, retries: int = DOWNLOAD_RETRIES) -> Optional[str]: attempt = 0 while attempt <= retries: path = download_pdf_once(url, dest_dir) if path: return path attempt += 1 return None def download_all_urls_concurrent(urls: List[str], dest_dir: Path, max_workers: int = DOWNLOAD_WORKERS) -> Dict[str, Optional[str]]: """ Download all URLs concurrently. Returns mapping url -> local_path (or None on failure). """ results: Dict[str, Optional[str]] = {} if not urls: return results # dedupe URLs while keeping order seen = set() uniq_urls = [] for u in urls: if u not in seen: seen.add(u) uniq_urls.append(u) with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: # schedule downloads future_to_url = {executor.submit(download_with_retries, url, dest_dir): url for url in uniq_urls} for fut in concurrent.futures.as_completed(future_to_url): url = future_to_url[fut] try: local = fut.result() except Exception: local = None results[url] = local return results # ---------------- Matching implementation (same logic as before) ---------------- PREFIXES = [ r'দখলদার মন্তব', r'দখলদার', r'মন্তব', r'ব \s* া \s* ক্ত', r'ব া ক্ত', r'বাক্তি', r'বাক্ত', r'বক্ত', r'দখ' ] PREFIX_RE = re.compile(r'^(?:' + "|".join(PREFIXES) + r')\s*', flags=re.I) HEADER_NOISE_RE = re.compile( r'(জ\.এল|জ\.এল নং|দাগর|দােগর|ম \s* াপ|ম াপ|Click Here|Live Data|Banglarbhumi|উল্লিখিত|খিত|Remarks|Nil|থানা|ব্লক|মৌজা)', flags=re.I ) SURNAME_HINTS = ["মন্ডল","মণ্ডল","হোসেন","ইসলাম","চৌধুরী","দাস","কুমার","রায়","রাজ","শর্মা"] PAIR_RE = re.compile(r'([\u0980-\u09FF\u0020\.\-]{2,160}?)\s+([০-৯0-9]{1,6}\s*/\s*[০-৯0-9]{0,6}|[০-৯0-9]{2,6})') NUM_MAP = str.maketrans("০১২৩৪৫৬৭৮৯","0123456789") def beng_to_ascii(s: str) -> str: return (s or "").translate(NUM_MAP) def clean_ctrl(s: str) -> str: if not s: return "" s = re.sub(r'[\x00-\x1F\x7F]+', ' ', s) s = re.sub(r'\s+', ' ', s) return s.strip() BENG_TO_LAT_MAP = { 'অ':'o','আ':'a','ই':'i','ঈ':'i','উ':'u','এ':'e','ঐ':'oi','ও':'o','ঔ':'ou', 'ক':'k','খ':'kh','গ':'g','ঘ':'gh','ঙ':'ng','চ':'ch','ছ':'chh','জ':'j','ঝ':'jh','ঞ':'n', 'ট':'t','ঠ':'th','ড':'d','ঢ':'dh','ণ':'n','ত':'t','থ':'th','দ':'d','ধ':'dh','ন':'n', 'প':'p','ফ':'ph','ব':'b','ভ':'bh','ম':'m','য':'y','র':'r','ল':'l','শ':'sh','ষ':'sh','স':'s','হ':'h', '্':'','া':'a','ি':'i','ী':'i','ু':'u','ূ':'u','ে':'e','ৈ':'oi','ো':'o','ৌ':'ou','ঁ':'n','ঃ':'' } def transliterate_bengali_to_latin(s: str) -> str: out = [] for ch in s: if ch in BENG_TO_LAT_MAP: out.append(BENG_TO_LAT_MAP[ch]) else: if re.match(r'[A-Za-z0-9 ]', ch): out.append(ch.lower()) else: out.append('') txt = "".join(out) txt = re.sub(r'\s+', ' ', txt).strip() return txt def fuzzy_score(a: str, b: str) -> float: if HAVE_RAPIDFUZZ: try: return float(fuzz.token_set_ratio(a,b)) except Exception: return float(fuzz.ratio(a,b)) else: return SequenceMatcher(None, a, b).ratio() * 100.0 def normalize_candidate_name(raw: str) -> str: s = (raw or "").strip() s = PREFIX_RE.sub('', s) s = HEADER_NOISE_RE.sub('', s) s = re.sub(r'[^ \u0980-\u09FFA-Za-z]', ' ', s) s = re.sub(r'\s{2,}', ' ', s).strip() return s def extract_pairs_from_pdf(pdf_path: str): """Extract candidate name + khatian-like number pairs from first page text.""" try: with pdfplumber.open(pdf_path) as pdf: page = pdf.pages[0] text = page.extract_text() or "" except Exception: return {"pdf": Path(pdf_path).name, "text_sample": "", "raw_pairs": [], "error": "pdf_open_failed"} text = clean_ctrl(text) raw_pairs = [] for m in PAIR_RE.finditer(text): raw_name = m.group(1).strip() raw_kh = m.group(2).strip() name = re.sub(r'\s{2,}', ' ', raw_name).strip(" -,.") kh = beng_to_ascii(raw_kh).replace(' ', '') raw_pairs.append({"name": name, "khatian": kh}) return {"pdf": Path(pdf_path).name, "text_sample": text, "raw_pairs": raw_pairs} def filter_and_normalize_pairs(raw_pairs): out = [] for p in raw_pairs: name = p.get("name","") kh = p.get("khatian","") name_norm = normalize_candidate_name(name) if not name_norm and not kh: continue if len(name_norm) < 2 and kh: continue out.append({"name": name_norm, "khatian": kh}) return out def match_user_input(pairs, target_khatian): return any(p.get("khatian") == target_khatian for p in pairs) def match_any_user_khatian(pairs, target_khatian_list): if not target_khatian_list: return False for kh in target_khatian_list: if any(p.get("khatian") == kh for p in pairs): return True return False def match_any_user_daag(pairs, text_sample, target_daag_list): if not target_daag_list: return False for d in target_daag_list: if any(p.get("khatian") == d for p in pairs): return True for d in target_daag_list: if re.search(r'\b' + re.escape(d) + r'\b', text_sample): return True return False def fuzzy_match_any_farmer(expected_farmers, candidates, khatian_present): best_overall = {"expected": "", "candidate": "", "score": 0.0, "translit": ""} if not expected_farmers or not candidates: return {**best_overall, "pass": False} for expected in expected_farmers: expected_norm = expected.strip() tnorm = re.sub(r'[^\w\s]', ' ', expected_norm).strip().lower() t_tokens = [tok for tok in re.split(r'\s+', tnorm) if tok] best_for_expected = {"candidate": "", "score": 0.0, "translit": ""} for cand in candidates: c = (cand or "").strip() if not c: continue cand_translit = transliterate_bengali_to_latin(c) if re.search(r'[\u0980-\u09FF]', c) else c.lower() cand_translit = re.sub(r'\s+', ' ', cand_translit).strip() c_tokens = [tok for tok in re.split(r'\s+', cand_translit) if tok] full_score = fuzzy_score(tnorm, cand_translit) token_best = 0.0 for tt in t_tokens: for ct in c_tokens: s = fuzzy_score(tt, ct) if s > token_best: token_best = s surname_boost = 0.0 if c_tokens: last = c_tokens[-1] for tt in t_tokens: if last and (tt == last or fuzzy_score(tt, last) > 85): surname_boost += 35.0 hint_boost = 0.0 for s in SURNAME_HINTS: if s in c: hint_boost += 10.0 combined = (0.6 * full_score) + (0.35 * token_best) + surname_boost + hint_boost combined += min(8, len(c_tokens)) if combined > best_for_expected["score"]: best_for_expected = {"candidate": c, "score": float(combined), "translit": cand_translit} if best_for_expected["score"] > best_overall["score"]: best_overall = {"expected": expected_norm, "candidate": best_for_expected["candidate"], "score": best_for_expected["score"], "translit": best_for_expected["translit"]} effective_threshold = KHATIAN_CONFIDENT_MIN_SCORE if khatian_present else FARMER_FUZZY_THRESHOLD best_overall["pass"] = best_overall["score"] >= effective_threshold return best_overall def strict_all_matched(payload_list, matched_list): if not payload_list: return True matched_set = set(matched_list or []) return all(item in matched_set for item in payload_list) def process_all(files: List[str], payload: Dict): """ Main orchestration: - Accepts a list of files (local paths or URLs). - Downloads all URLs concurrently (mapping url -> local path or None). - Processes each local file (or reports errors for missing/failed downloads). """ out = {"results": [], "summary": {}} payload_daag_list = payload.get("daag") or [] payload_khatian_list = payload.get("khatian") or [] payload_farmer_list = payload.get("farmer") or [] daag_matched_items = [] khatian_matched_items = [] farmer_matched_items = [] # separate URLs and non-URLs urls = [f for f in files if is_url(f)] non_urls = [f for f in files if not is_url(f)] # download URLs concurrently url_to_local = download_all_urls_concurrent(urls, DOWNLOAD_DIR, max_workers=DOWNLOAD_WORKERS) if urls else {} # build final list of local sources to process (preserve original order) final_sources: List[Dict] = [] for f in files: if is_url(f): local = url_to_local.get(f) final_sources.append({"orig": f, "local": local, "is_url": True}) else: final_sources.append({"orig": f, "local": f, "is_url": False}) # process each source for src in final_sources: orig = src["orig"] local = src["local"] if src["is_url"]: if not local: # download failed out["results"].append({ "pdf": Path(orig).name, "source": orig, "error": "download_failed", "path_or_url": orig }) continue # local should exist now if not Path(local).exists(): out["results"].append({ "pdf": Path(orig).name, "source": local, "error": "file_not_found", "path_or_url": orig }) continue rec = extract_pairs_from_pdf(local) if rec.get("error"): out["results"].append({ "pdf": rec.get("pdf"), "source": local, "error": rec.get("error") }) continue raw_pairs = rec["raw_pairs"] pairs = filter_and_normalize_pairs(raw_pairs) text_sample = rec.get("text_sample","") daag_present = match_any_user_daag(pairs, text_sample, payload_daag_list) khatian_present = match_any_user_khatian(pairs, payload_khatian_list) candidates = [p["name"] for p in pairs if p.get("name")] fam_best = fuzzy_match_any_farmer(payload_farmer_list, candidates, khatian_present) if daag_present: for d in payload_daag_list: if d not in daag_matched_items: if any(p.get("khatian") == d for p in pairs) or re.search(r'\b'+re.escape(d)+r'\b', text_sample): daag_matched_items.append(d) if khatian_present: for k in payload_khatian_list: if k not in khatian_matched_items and any(p.get("khatian")==k for p in pairs): khatian_matched_items.append(k) if fam_best.get("pass"): em = fam_best.get("expected") if em and em not in farmer_matched_items: farmer_matched_items.append(em) out_item = { "pdf": rec["pdf"], "source": local, "pairs": pairs, "matches": { "daag_expected": payload_daag_list, "daag_match": {"present": daag_present}, "khatian_expected": payload_khatian_list, "khatian_match": {"present": khatian_present}, "farmer_expected": payload_farmer_list, "farmer_match": { "expected_matched": fam_best.get("expected",""), "extracted": fam_best.get("candidate",""), "translit": fam_best.get("translit",""), "score": fam_best.get("score", 0.0), "pass": fam_best.get("pass", False) } } } out["results"].append(out_item) strict_daag_ok = strict_all_matched(payload_daag_list, daag_matched_items) strict_khatian_ok = strict_all_matched(payload_khatian_list, khatian_matched_items) strict_farmer_ok = strict_all_matched(payload_farmer_list, farmer_matched_items) status = "ACCEPTED" if (strict_daag_ok and strict_khatian_ok and strict_farmer_ok) else "REJECTED" missing_daag = [d for d in payload_daag_list if d not in daag_matched_items] missing_khatian = [k for k in payload_khatian_list if k not in khatian_matched_items] missing_farmer = [f for f in payload_farmer_list if f not in farmer_matched_items] overall = { "daag_matched_items": daag_matched_items, "khatian_matched_items": khatian_matched_items, "farmer_matched_items": farmer_matched_items } out["summary"] = { "files_checked": len(files), "payload": payload, "overall_match": overall, "missing": { "daag": missing_daag, "khatian": missing_khatian, "farmer": missing_farmer }, "strict_checks": { "strict_daag_ok": strict_daag_ok, "strict_khatian_ok": strict_khatian_ok, "strict_farmer_ok": strict_farmer_ok }, "status": status } return out # ---------------- FastAPI endpoints ---------------- @app.get("/health") def health(): return {"status": "ok"} @app.post("/match") async def match_endpoint( payload_json: Optional[str] = Form(None), upload_files: Optional[List[UploadFile]] = File(None) ): """ Multipart form entrypoint: - payload_json: JSON string - upload_files: optional list of files """ if payload_json: try: payload_obj = json.loads(payload_json) except Exception as e: raise HTTPException(status_code=400, detail=f"invalid payload_json: {e}") else: raise HTTPException(status_code=400, detail="Please POST JSON to /match_json or send payload_json form field.") files_to_process = [] if upload_files: for up in upload_files: dest = UPLOAD_DIR / up.filename with dest.open("wb") as f: shutil.copyfileobj(up.file, f) files_to_process.append(str(dest.resolve())) files_from_payload = payload_obj.get("files") or [] for p in files_from_payload: files_to_process.append(p) if not files_to_process: files_to_process = DEFAULT_FILES.copy() try: pay = MatchPayload( daag=payload_obj.get("daag", []), khatian=payload_obj.get("khatian", []), farmer=payload_obj.get("farmer", []), files=[] ) except Exception as e: raise HTTPException(status_code=400, detail=f"payload validation error: {e}") payload_for_proc = { "daag": pay.daag, "khatian": pay.khatian, "farmer": pay.farmer } results = process_all(files_to_process, payload_for_proc) return JSONResponse(content=results) @app.post("/match_json") async def match_json(payload: MatchPayload): files_to_process = [] if payload.files: for p in payload.files: files_to_process.append(p) if not files_to_process: files_to_process = DEFAULT_FILES.copy() payload_for_proc = { "daag": payload.daag, "khatian": payload.khatian, "farmer": payload.farmer } results = process_all(files_to_process, payload_for_proc) return JSONResponse(content=results) ######################################################################## # #!/usr/bin/env python3 # """ # table_matcher_v4_strict_missing.py # - Accepts multiple expected values in PAYLOAD: daag, khatian, farmer (lists). # - Extracts (name, khatian) pairs from first page of each PDF (pdfplumber). # - Option B for farmer matching: if any expected khatian is present in a file, # farmer matching for that file uses the permissive threshold. # - STRICT overall decision: ALL user-supplied values (every daag, every khatian, # every farmer) must be found/matched somewhere across the checked PDF files # for final status to be ACCEPTED. Otherwise REJECTED. # - Summary includes lists of missing values per category. # - Writes JSON to /mnt/data/match_results_v4_optionB_strict_missing.json # """ # from pathlib import Path # import re # import json # from difflib import SequenceMatcher # # optional: use rapidfuzz if installed for better fuzzy matching # try: # from rapidfuzz import fuzz # HAVE_RAPIDFUZZ = True # except Exception: # HAVE_RAPIDFUZZ = False # import pdfplumber # # ---------------- CONFIG ---------------- # FILES = [ # "371(Plot No)_Suahanta Mondal.pdf", # "348__South Gobindapur__SAINTHIA.pdf", # "345__South Gobindapur__SAINTHIA.pdf", # "1764746873181-519NetureeSAINTHIA.pdf" # ] # # Example payload (replace with incoming payload) # PAYLOAD = { # "daag": ["371","519"], # "khatian": ["1077","61","6765","152/1"], # "farmer": ["Indrani Mondal","Sushanta Mandal","Suman Chatterjee"] # } # # Matching thresholds and policy # REQUIRE_KHATIAN_FOR_FARMER_MATCH = True # FARMER_FUZZY_THRESHOLD = 85.0 # strict baseline when khatian not present # KHATIAN_CONFIDENT_MIN_SCORE = 60.0 # permissive when khatian present in that file # # ---------------- patterns & helpers ---------------- # PREFIXES = [ # r'দখলদার মন্তব', r'দখলদার', r'মন্তব', # r'ব \s* া \s* ক্ত', r'ব া ক্ত', r'বাক্তি', r'বাক্ত', r'বক্ত', r'দখ' # ] # PREFIX_RE = re.compile(r'^(?:' + "|".join(PREFIXES) + r')\s*', flags=re.I) # HEADER_NOISE_RE = re.compile( # r'(জ\.এল|জ\.এল নং|দাগর|দােগর|ম \s* াপ|ম াপ|Click Here|Live Data|Banglarbhumi|উল্লিখিত|খিত|Remarks|Nil|থানা|ব্লক|মৌজা)', # flags=re.I # ) # SURNAME_HINTS = ["মন্ডল","মণ্ডল","হোসেন","ইসলাম","চৌধুরী","দাস","কুমার","রায়","রাজ","শর্মা"] # # pair regex: some Bengali text before a number-like token (khatian/daag) # PAIR_RE = re.compile(r'([\u0980-\u09FF\u0020\.\-]{2,160}?)\s+([০-৯0-9]{1,6}\s*/\s*[০-৯0-9]{0,6}|[০-৯0-9]{2,6})') # NUM_MAP = str.maketrans("০১২৩৪৫৬৭৮৯","0123456789") # def beng_to_ascii(s: str) -> str: # return (s or "").translate(NUM_MAP) # def clean_ctrl(s: str) -> str: # if not s: # return "" # s = re.sub(r'[\x00-\x1F\x7F]+', ' ', s) # s = re.sub(r'\s+', ' ', s) # return s.strip() # # coarse Bengali->Latin map (best-effort) # BENG_TO_LAT_MAP = { # 'অ':'o','আ':'a','ই':'i','ঈ':'i','উ':'u','এ':'e','ঐ':'oi','ও':'o','ঔ':'ou', # 'ক':'k','খ':'kh','গ':'g','ঘ':'gh','ঙ':'ng','চ':'ch','ছ':'chh','জ':'j','ঝ':'jh','ঞ':'n', # 'ট':'t','ঠ':'th','ড':'d','ঢ':'dh','ণ':'n','ত':'t','থ':'th','দ':'d','ধ':'dh','ন':'n', # 'প':'p','ফ':'ph','ব':'b','ভ':'bh','ম':'m','য':'y','র':'r','ল':'l','শ':'sh','ষ':'sh','স':'s','হ':'h', # '্':'','া':'a','ি':'i','ী':'i','ু':'u','ূ':'u','ে':'e','ৈ':'oi','ো':'o','ৌ':'ou','ঁ':'n','ঃ':'' # } # def transliterate_bengali_to_latin(s: str) -> str: # out = [] # for ch in s: # if ch in BENG_TO_LAT_MAP: # out.append(BENG_TO_LAT_MAP[ch]) # else: # if re.match(r'[A-Za-z0-9 ]', ch): # out.append(ch.lower()) # else: # out.append('') # txt = "".join(out) # txt = re.sub(r'\s+', ' ', txt).strip() # return txt # def fuzzy_score(a: str, b: str) -> float: # if HAVE_RAPIDFUZZ: # try: # return float(fuzz.token_set_ratio(a,b)) # except Exception: # return float(fuzz.ratio(a,b)) # else: # return SequenceMatcher(None, a, b).ratio() * 100.0 # def normalize_candidate_name(raw: str) -> str: # s = (raw or "").strip() # s = PREFIX_RE.sub('', s) # s = HEADER_NOISE_RE.sub('', s) # s = re.sub(r'[^ \u0980-\u09FFA-Za-z]', ' ', s) # s = re.sub(r'\s{2,}', ' ', s).strip() # return s # # ---------------- extraction ---------------- # def extract_pairs_from_pdf(pdf_path: str): # """Extract candidate name + khatian-like number pairs from first page text.""" # with pdfplumber.open(pdf_path) as pdf: # page = pdf.pages[0] # text = page.extract_text() or "" # text = clean_ctrl(text) # raw_pairs = [] # for m in PAIR_RE.finditer(text): # raw_name = m.group(1).strip() # raw_kh = m.group(2).strip() # name = re.sub(r'\s{2,}', ' ', raw_name).strip(" -,.") # kh = beng_to_ascii(raw_kh).replace(' ', '') # raw_pairs.append({"name": name, "khatian": kh}) # return {"pdf": Path(pdf_path).name, "text_sample": text, "raw_pairs": raw_pairs} # def filter_and_normalize_pairs(raw_pairs): # out = [] # for p in raw_pairs: # name = p.get("name","") # kh = p.get("khatian","") # name_norm = normalize_candidate_name(name) # if not name_norm and not kh: # continue # if len(name_norm) < 2 and kh: # continue # out.append({"name": name_norm, "khatian": kh}) # return out # # ---------------- matching helpers ---------------- # def match_user_input(pairs, target_khatian): # """Return True if any pair's khatian equals target_khatian (exact).""" # return any(p.get("khatian") == target_khatian for p in pairs) # def match_any_user_khatian(pairs, target_khatian_list): # if not target_khatian_list: # return False # for kh in target_khatian_list: # if any(p.get("khatian") == kh for p in pairs): # return True # return False # def match_any_user_daag(pairs, text_sample, target_daag_list): # if not target_daag_list: # return False # for d in target_daag_list: # if any(p.get("khatian") == d for p in pairs): # return True # for d in target_daag_list: # if re.search(r'\b' + re.escape(d) + r'\b', text_sample): # return True # return False # def fuzzy_match_any_farmer(expected_farmers, candidates, khatian_present): # """ # Try all expected farmers; return the best match across them. # Returns dict: {expected, candidate, score, translit, pass} # """ # best_overall = {"expected": "", "candidate": "", "score": 0.0, "translit": ""} # if not expected_farmers or not candidates: # return {**best_overall, "pass": False} # for expected in expected_farmers: # expected_norm = expected.strip() # tnorm = re.sub(r'[^\w\s]', ' ', expected_norm).strip().lower() # t_tokens = [tok for tok in re.split(r'\s+', tnorm) if tok] # best_for_expected = {"candidate": "", "score": 0.0, "translit": ""} # for cand in candidates: # c = (cand or "").strip() # if not c: # continue # cand_translit = transliterate_bengali_to_latin(c) if re.search(r'[\u0980-\u09FF]', c) else c.lower() # cand_translit = re.sub(r'\s+', ' ', cand_translit).strip() # c_tokens = [tok for tok in re.split(r'\s+', cand_translit) if tok] # full_score = fuzzy_score(tnorm, cand_translit) # token_best = 0.0 # for tt in t_tokens: # for ct in c_tokens: # s = fuzzy_score(tt, ct) # if s > token_best: # token_best = s # surname_boost = 0.0 # if c_tokens: # last = c_tokens[-1] # for tt in t_tokens: # if last and (tt == last or fuzzy_score(tt, last) > 85): # surname_boost += 35.0 # hint_boost = 0.0 # for s in SURNAME_HINTS: # if s in c: # hint_boost += 10.0 # combined = (0.6 * full_score) + (0.35 * token_best) + surname_boost + hint_boost # combined += min(8, len(c_tokens)) # if combined > best_for_expected["score"]: # best_for_expected = {"candidate": c, "score": float(combined), "translit": cand_translit} # if best_for_expected["score"] > best_overall["score"]: # best_overall = {"expected": expected_norm, # "candidate": best_for_expected["candidate"], # "score": best_for_expected["score"], # "translit": best_for_expected["translit"]} # effective_threshold = KHATIAN_CONFIDENT_MIN_SCORE if khatian_present else FARMER_FUZZY_THRESHOLD # best_overall["pass"] = best_overall["score"] >= effective_threshold # return best_overall # # ---------------- orchestration ---------------- # def strict_all_matched(payload_list, matched_list): # """ # Return True if EVERY item in payload_list is present in matched_list. # If payload_list is empty -> consider it satisfied (True). # """ # if not payload_list: # return True # matched_set = set(matched_list or []) # return all(item in matched_set for item in payload_list) # def process_all(files, payload): # out = {"results": [], "summary": {}} # payload_daag_list = payload.get("daag") or [] # payload_khatian_list = payload.get("khatian") or [] # payload_farmer_list = payload.get("farmer") or [] # # accumulators for matched payload items across ANY file # daag_matched_items = [] # khatian_matched_items = [] # farmer_matched_items = [] # store the expected farmer strings that were matched # for fp in files: # rec = extract_pairs_from_pdf(fp) # raw_pairs = rec["raw_pairs"] # pairs = filter_and_normalize_pairs(raw_pairs) # text_sample = rec.get("text_sample","") # # detect presence of any expected daag/khatian in this file # daag_present = match_any_user_daag(pairs, text_sample, payload_daag_list) # khatian_present = match_any_user_khatian(pairs, payload_khatian_list) # # build candidate list (cleaned names) # candidates = [p["name"] for p in pairs if p.get("name")] # # find best farmer among all expected farmers for this file # fam_best = fuzzy_match_any_farmer(payload_farmer_list, candidates, khatian_present) # # record matched payload items discovered in this file # if daag_present: # for d in payload_daag_list: # if d not in daag_matched_items: # if any(p.get("khatian") == d for p in pairs) or re.search(r'\b'+re.escape(d)+r'\b', text_sample): # daag_matched_items.append(d) # if khatian_present: # for k in payload_khatian_list: # if k not in khatian_matched_items and any(p.get("khatian")==k for p in pairs): # khatian_matched_items.append(k) # if fam_best.get("pass"): # em = fam_best.get("expected") # if em and em not in farmer_matched_items: # farmer_matched_items.append(em) # out_item = { # "pdf": rec["pdf"], # "pairs": pairs, # "matches": { # "daag_expected": payload_daag_list, # "daag_match": {"present": daag_present}, # "khatian_expected": payload_khatian_list, # "khatian_match": {"present": khatian_present}, # "farmer_expected": payload_farmer_list, # "farmer_match": { # "expected_matched": fam_best.get("expected",""), # "extracted": fam_best.get("candidate",""), # "translit": fam_best.get("translit",""), # "score": fam_best.get("score", 0.0), # "pass": fam_best.get("pass", False) # } # } # } # out["results"].append(out_item) # # strict checks: every payload item must be matched somewhere across files # strict_daag_ok = strict_all_matched(payload_daag_list, daag_matched_items) # strict_khatian_ok = strict_all_matched(payload_khatian_list, khatian_matched_items) # strict_farmer_ok = strict_all_matched(payload_farmer_list, farmer_matched_items) # status = "ACCEPTED" if (strict_daag_ok and strict_khatian_ok and strict_farmer_ok) else "REJECTED" # # compute missing lists # missing_daag = [d for d in payload_daag_list if d not in daag_matched_items] # missing_khatian = [k for k in payload_khatian_list if k not in khatian_matched_items] # missing_farmer = [f for f in payload_farmer_list if f not in farmer_matched_items] # overall = { # "daag_matched_items": daag_matched_items, # "khatian_matched_items": khatian_matched_items, # "farmer_matched_items": farmer_matched_items # } # out["summary"] = { # "files_checked": len(files), # "payload": payload, # "overall_match": overall, # "missing": { # "daag": missing_daag, # "khatian": missing_khatian, # "farmer": missing_farmer # }, # "strict_checks": { # "strict_daag_ok": strict_daag_ok, # "strict_khatian_ok": strict_khatian_ok, # "strict_farmer_ok": strict_farmer_ok # }, # "status": status # } # return out # # ---------------- run ---------------- # if __name__ == "__main__": # results = process_all(FILES, PAYLOAD) # outpath = Path("match_results_v4_optionB_strict_missing.json") # outpath.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") # print(json.dumps(results, ensure_ascii=False, indent=2)) # print("Saved ->", outpath) # # #!/usr/bin/env python3 # # """ # # table_matcher_with_fuzzy_v4_optionB_multi_strict.py # # - Supports multiple expected values in PAYLOAD: # # "daag": [...], "khatian": [...], "farmer": [...] # # - Option B: if any expected khatian is present in the file, farmer matching uses permissive threshold. # # - STRICT rule: ALL user-provided payload values must be found across the files to return "ACCEPTED". # # - Writes /mnt/data/match_results_v4_optionB_multi_strict.json # # """ # # from pathlib import Path # # import re, json # # from difflib import SequenceMatcher # # # optional: rapidfuzz for better fuzzy matching; fallback to difflib if missing # # try: # # from rapidfuzz import fuzz # # HAVE_RAPIDFUZZ = True # # except Exception: # # HAVE_RAPIDFUZZ = False # # import pdfplumber # # # ---------------- CONFIG ---------------- # # FILES = [ # # "371(Plot No)_Suahanta Mondal.pdf", # # "348__South Gobindapur__SAINTHIA.pdf", # # "345__South Gobindapur__SAINTHIA.pdf", # # "1764746873181-519NetureeSAINTHIA.pdf" # # ] # # # Example multi-value payload (replace with actual input) # # PAYLOAD = { # # "daag": ["371", "519"], # # "khatian": ["1077", "61", "6765", "152/1"], # # "farmer": ["Indrani Mondal", "Sushanta Mandal", "Suman Chatterjee"] # # } # # # Matching configuration # # REQUIRE_KHATIAN_FOR_FARMER_MATCH = True # # FARMER_FUZZY_THRESHOLD = 85.0 # strict baseline when no khatian matched in file # # KHATIAN_CONFIDENT_MIN_SCORE = 60.0 # permissive threshold when khatian matched in file # # # ---------------- patterns / helpers ---------------- # # PREFIXES = [ # # r'দখলদার মন্তব', r'দখলদার', r'মন্তব', # # r'ব \s* া \s* ক্ত', r'ব া ক্ত', r'বাক্তি', r'বাক্ত', r'বক্ত', r'দখ' # # ] # # PREFIX_RE = re.compile(r'^(?:' + "|".join(PREFIXES) + r')\s*', flags=re.I) # # HEADER_NOISE_RE = re.compile( # # r'(জ\.এল|জ\.এল নং|দাগর|দােগর|ম \s* াপ|ম াপ|Click Here|Live Data|Banglarbhumi|উল্লিখিত|খিত|Remarks|Nil|থানা|ব্লক|মৌজা)', # # flags=re.I # # ) # # SURNAME_HINTS = ["মন্ডল","মণ্ডল","হোসেন","ইসলাম","চৌধুরী","দাস","কুমার","রায়","রাজ","শর্মা"] # # # match name-like chunk followed by khatian-like number # # PAIR_RE = re.compile(r'([\u0980-\u09FF\u0020\.\-]{2,160}?)\s+([০-৯0-9]{1,6}\s*/\s*[০-৯0-9]{0,6}|[০-৯0-9]{2,6})') # # NUM_MAP = str.maketrans("০১২৩৪৫৬৭৮৯","0123456789") # # def beng_to_ascii(s: str) -> str: # # return (s or "").translate(NUM_MAP) # # def clean_ctrl(s: str) -> str: # # if not s: return "" # # s = re.sub(r'[\x00-\x1F\x7F]+',' ', s) # # s = re.sub(r'\s+',' ', s) # # return s.strip() # # # coarse transliteration map (Bengali -> Latin) — best-effort # # BENG_TO_LAT_MAP = { # # 'অ':'o','আ':'a','ই':'i','ঈ':'i','উ':'u','এ':'e','ঐ':'oi','ও':'o','ঔ':'ou', # # 'ক':'k','খ':'kh','গ':'g','ঘ':'gh','ঙ':'ng','চ':'ch','ছ':'chh','জ':'j','ঝ':'jh','ঞ':'n', # # 'ট':'t','ঠ':'th','ড':'d','ঢ':'dh','ণ':'n','ত':'t','থ':'th','দ':'d','ধ':'dh','ন':'n', # # 'প':'p','ফ':'ph','ব':'b','ভ':'bh','ম':'m','য':'y','র':'r','ল':'l','শ':'sh','ষ':'sh','স':'s','হ':'h', # # '্':'','া':'a','ি':'i','ী':'i','ু':'u','ূ':'u','ে':'e','ৈ':'oi','ো':'o','ৌ':'ou','ঁ':'n','ঃ':'' # # } # # def transliterate_bengali_to_latin(s: str) -> str: # # out = [] # # for ch in s: # # if ch in BENG_TO_LAT_MAP: # # out.append(BENG_TO_LAT_MAP[ch]) # # else: # # if re.match(r'[A-Za-z0-9 ]', ch): # # out.append(ch.lower()) # # else: # # out.append('') # # txt = "".join(out) # # txt = re.sub(r'\s+',' ', txt).strip() # # return txt # # def fuzzy_score(a: str, b: str) -> float: # # if HAVE_RAPIDFUZZ: # # try: # # return float(fuzz.token_set_ratio(a,b)) # # except Exception: # # return float(fuzz.ratio(a,b)) # # else: # # return SequenceMatcher(None, a, b).ratio() * 100.0 # # def normalize_candidate_name(raw: str) -> str: # # s = (raw or "").strip() # # s = PREFIX_RE.sub('', s) # # s = HEADER_NOISE_RE.sub('', s) # # s = re.sub(r'[^ \u0980-\u09FFA-Za-z]', ' ', s) # # s = re.sub(r'\s{2,}', ' ', s).strip() # # return s # # # ---------------- extraction ---------------- # # def extract_pairs_from_pdf(pdf_path: str): # # """Return raw_pairs list (name, khatian) and text_sample.""" # # with pdfplumber.open(pdf_path) as pdf: # # page = pdf.pages[0] # # text = page.extract_text() or "" # # text = clean_ctrl(text) # # raw_pairs = [] # # for m in PAIR_RE.finditer(text): # # raw_name = m.group(1).strip() # # raw_kh = m.group(2).strip() # # name = re.sub(r'\s{2,}',' ', raw_name).strip(" -,.") # # kh = beng_to_ascii(raw_kh).replace(' ','') # # raw_pairs.append({"name": name, "khatian": kh}) # # return {"pdf": Path(pdf_path).name, "text_sample": text, "raw_pairs": raw_pairs} # # def filter_and_normalize_pairs(raw_pairs): # # out=[] # # for p in raw_pairs: # # name = p.get("name","") # # kh = p.get("khatian","") # # name_norm = normalize_candidate_name(name) # # if not name_norm and not kh: # # continue # # if len(name_norm) < 2 and kh: # # continue # # out.append({"name": name_norm, "khatian": kh}) # # return out # # # ---------------- matching helpers ---------------- # # # matches exact khatian within pairs # # def match_any_user_khatian(pairs, target_khatian_list): # # if not target_khatian_list: # # return False # # for kh in target_khatian_list: # # if any(p.get("khatian")==kh for p in pairs): # # return True # # return False # # # match any daag — check pair khatians and fallback to searching text_sample # # def match_any_user_daag(pairs, text_sample, target_daag_list): # # if not target_daag_list: # # return False # # for d in target_daag_list: # # if any(p.get("khatian")==d for p in pairs): # # return True # # if re.search(r'\b' + re.escape(d) + r'\b', text_sample): # # return True # # return False # # # EXACT function requested earlier — used for single checks if needed # # def match_user_input(pairs, target_khatian): # # return any(p.get("khatian")==target_khatian for p in pairs) # # # fuzzy-match across multiple expected farmers: returns best expected matched for this file # # def fuzzy_match_any_farmer(expected_farmers, candidates, khatian_present): # # best_overall = {"expected":"", "candidate":"", "score":0.0, "translit":""} # # if not expected_farmers or not candidates: # # return {**best_overall, "pass": False} # # for expected in expected_farmers: # # expected_norm = expected.strip() # # tnorm = re.sub(r'[^\w\s]',' ', expected_norm).strip().lower() # # t_tokens = [tok for tok in re.split(r'\s+', tnorm) if tok] # # best_for_expected = {"candidate":"", "score":0.0, "translit":""} # # for cand in candidates: # # c = (cand or "").strip() # # if not c: continue # # cand_translit = transliterate_bengali_to_latin(c) if re.search(r'[\u0980-\u09FF]', c) else c.lower() # # cand_translit = re.sub(r'\s+',' ', cand_translit).strip() # # c_tokens = [tok for tok in re.split(r'\s+', cand_translit) if tok] # # full_score = fuzzy_score(tnorm, cand_translit) # # token_best = 0.0 # # for tt in t_tokens: # # for ct in c_tokens: # # s = fuzzy_score(tt, ct) # # if s > token_best: # # token_best = s # # surname_boost = 0.0 # # if c_tokens: # # last = c_tokens[-1] # # for tt in t_tokens: # # if last and (tt == last or fuzzy_score(tt, last) > 85): # # surname_boost += 35.0 # # hint_boost = 0.0 # # for s in SURNAME_HINTS: # # if s in c: # # hint_boost += 10.0 # # combined = (0.6 * full_score) + (0.35 * token_best) + surname_boost + hint_boost # # combined += min(8, len(c_tokens)) # # if combined > best_for_expected["score"]: # # best_for_expected = {"candidate": c, "score": float(combined), "translit": cand_translit} # # # if best_for_expected better than global, choose it # # if best_for_expected["score"] > best_overall["score"]: # # best_overall = {"expected": expected_norm, # # "candidate": best_for_expected["candidate"], # # "score": best_for_expected["score"], # # "translit": best_for_expected["translit"]} # # # pass decision depends on whether khatian_present in file (Option B) # # effective_threshold = KHATIAN_CONFIDENT_MIN_SCORE if khatian_present else FARMER_FUZZY_THRESHOLD # # best_overall["pass"] = best_overall["score"] >= effective_threshold # # return best_overall # # # ---------------- orchestrator ---------------- # # def all_payload_values_matched(strict_payload_list, matched_items_list): # # """ # # strict check: every element in strict_payload_list must be present in matched_items_list. # # (Both lists contain strings; perform direct equality) # # """ # # if not strict_payload_list: # # # if user provided empty list, treat as matched by default # # return True # # matched_set = set((m or "").strip() for m in matched_items_list) # # for item in strict_payload_list: # # if (item or "").strip() not in matched_set: # # return False # # return True # # def process_all(files, payload): # # out = {"results": [], "summary": {}} # # payload_daag_list = payload.get("daag") or [] # # payload_khatian_list = payload.get("khatian") or [] # # payload_farmer_list = payload.get("farmer") or [] # # # Collect matched items across all files # # overall_daag_matched_items = [] # # overall_khatian_matched_items = [] # # overall_farmer_matched_items = [] # these are EXPECTED farmer strings that were matched # # for fp in files: # # rec = extract_pairs_from_pdf(fp) # # raw_pairs = rec["raw_pairs"] # # pairs = filter_and_normalize_pairs(raw_pairs) # # text_sample = rec.get("text_sample","") # # # file-level matches # # daag_pass = match_any_user_daag(pairs, text_sample, payload_daag_list) # # kh_pass = match_any_user_khatian(pairs, payload_khatian_list) # # # candidates (cleaned name strings) used for farmer fuzzy matching # # candidates = [p["name"] for p in pairs if p.get("name")] # # fam_best = fuzzy_match_any_farmer(payload_farmer_list, candidates, kh_pass) # # # record which specific payload daag/khatian matched in this file # # file_daag_matched = [] # # for d in payload_daag_list: # # found = False # # if any(p.get("khatian")==d for p in pairs): # # found = True # # elif re.search(r'\b' + re.escape(d) + r'\b', text_sample): # # found = True # # if found: # # file_daag_matched.append(d) # # if d not in overall_daag_matched_items: # # overall_daag_matched_items.append(d) # # file_khatian_matched = [] # # for k in payload_khatian_list: # # if any(p.get("khatian")==k for p in pairs): # # file_khatian_matched.append(k) # # if k not in overall_khatian_matched_items: # # overall_khatian_matched_items.append(k) # # # if fam_best.pass True, that means fam_best["expected"] (one of payload farmers) was matched here # # file_farmer_matched_expected = fam_best.get("expected") if fam_best.get("pass") else "" # # if file_farmer_matched_expected: # # if file_farmer_matched_expected not in overall_farmer_matched_items: # # overall_farmer_matched_items.append(file_farmer_matched_expected) # # out_item = { # # "pdf": rec["pdf"], # # "pairs": pairs, # # "files_matches": { # # "daag_match": {"pass": daag_pass, "matched_items": file_daag_matched}, # # "khatian_match": {"pass": kh_pass, "matched_items": file_khatian_matched}, # # "farmer_match": { # # "expected_matched": fam_best.get("expected",""), # # "extracted": fam_best.get("candidate",""), # # "translit": fam_best.get("translit",""), # # "score": fam_best.get("score",0.0), # # "pass": fam_best.get("pass", False) # # } # # } # # } # # out["results"].append(out_item) # # # STRICT overall checks: all payload items must be present in the aggregated matched items # # strict_daag_ok = all_payload_values_matched(payload_daag_list, overall_daag_matched_items) # # strict_khatian_ok = all_payload_values_matched(payload_khatian_list, overall_khatian_matched_items) # # strict_farmer_ok = all_payload_values_matched(payload_farmer_list, overall_farmer_matched_items) # # status = "ACCEPTED" if (strict_daag_ok and strict_khatian_ok and strict_farmer_ok) else "REJECTED" # # out["summary"] = { # # "files_checked": len(files), # # "payload": payload, # # "overall_match": { # # "daag_match": bool(overall_daag_matched_items), # # "khatian_match": bool(overall_khatian_matched_items), # # "farmer_match": bool(overall_farmer_matched_items), # # "daag_matched_items": overall_daag_matched_items, # # "khatian_matched_items": overall_khatian_matched_items, # # "farmer_matched_items": overall_farmer_matched_items # # }, # # "strict_checks": { # # "strict_daag_ok": strict_daag_ok, # # "strict_khatian_ok": strict_khatian_ok, # # "strict_farmer_ok": strict_farmer_ok # # }, # # "status": status # # } # # return out # # # ---------------- main ---------------- # # if __name__ == "__main__": # # results = process_all(FILES, PAYLOAD) # # outpath = Path("match_results_v4_optionB_multi_strict.json") # # outpath.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") # # print(json.dumps(results, ensure_ascii=False, indent=2)) # # print("Saved ->", outpath)