jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/handwritten-npi-list-updated (pull request #850)

Browse Source 
fix ocr issue in list of NPIs

* fix ocr issue in list od NPIs

* pipeline error fixed


Approved-by: Katon Minhas
This commit is contained in:
Mayank Aamseek
2026-01-27 17:28:46 +00:00
committed by Katon Minhas
parent afb6d5185d
commit 929ff4abf8
2 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/constants/regex_patterns.py
+3 -2
View File
@@ -40,7 +40,7 @@ NPI_PATTERN = r"\b(?:\d\s*){10}\b"
TIN_OCR_PATTERN = r"\b[O0lI1S5G6B8gbo\d]{9}\b|\b[O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S5G6B8gbo\d]{7}\b|\b[O0lI1S5G6B8gbo\d]{3}[-.][O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S5G6B8gbo\d]{4}\b" TIN_OCR_PATTERN = r"\b[O0lI1S5G6B8gbo\d]{9}\b|\b[O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S5G6B8gbo\d]{7}\b|\b[O0lI1S5G6B8gbo\d]{3}[-.][O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S5G6B8gbo\d]{4}\b"
# OCR-Correctable NPI Pattern # OCR-Correctable NPI Pattern
# Strategy: Match 10-character sequences containing digits and OCR-prone letters, # Strategy: Match 10-character sequences containing digits and OCR-prone letters; also Matches "chained" NPIs separated by '1', 'l', or 'I'.
# then apply corrections post-match to validate it becomes a proper 10-digit NPI. # then apply corrections post-match to validate it becomes a proper 10-digit NPI.
# #
# Examples that WILL be captured and corrected: # Examples that WILL be captured and corrected:
@@ -48,6 +48,7 @@ TIN_OCR_PATTERN = r"\b[O0lI1S5G6B8gbo\d]{9}\b|\b[O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S
# - "l234567890" → "1234567890" (l at start) # - "l234567890" → "1234567890" (l at start)
# - "12345O7890" → "1234507890" (O in middle) # - "12345O7890" → "1234507890" (O in middle)
# - "123456789l" → "1234567891" (l at end) # - "123456789l" → "1234567891" (l at end)
# - "123456789011234567890" (seen as NPI + "1" + NPI)
# #
# Examples that will NOT match: # Examples that will NOT match:
# - "123456789" (only 9 characters) # - "123456789" (only 9 characters)
@@ -57,7 +58,7 @@ TIN_OCR_PATTERN = r"\b[O0lI1S5G6B8gbo\d]{9}\b|\b[O0lI1S5G6B8gbo\d]{2}[-.][O0lI1S
# Note: NPIs are always unformatted 10-digit numbers, so no hyphen/dot variations. # Note: NPIs are always unformatted 10-digit numbers, so no hyphen/dot variations.
# Validation happens post-match by applying OCR_SUBSTITUTIONS and checking # Validation happens post-match by applying OCR_SUBSTITUTIONS and checking
# that result is exactly 10 digits. # that result is exactly 10 digits.
NPI_OCR_PATTERN = r"\b[O0lI1S5G6B8gbo\d]{10}\b" NPI_OCR_PATTERN = r"\b(?:[O0lI1S5G6B8gbo\d]{10}[1lI])*[O0lI1S5G6B8gbo\d]{10}\b"
# OCR Substitution Mapping # OCR Substitution Mapping
OCR_SUBSTITUTIONS = { OCR_SUBSTITUTIONS = {
src/pipelines/shared/extraction/tin_npi_funcs.py
+16
View File
@@ -66,6 +66,22 @@ def get_all_matches_with_ocr(
# Get OCR-correctable candidates (even if we found exact matches) # Get OCR-correctable candidates (even if we found exact matches)
ocr_candidates = get_all_matches(text, ocr_pattern) ocr_candidates = get_all_matches(text, ocr_pattern)
for candidate in ocr_candidates: for candidate in ocr_candidates:
# Checks if length follows pattern: 10*N + (N-1) -> (len + 1) % 11 == 0
if (
identifier_type == "NPI"
and len(candidate) > 10
and (len(candidate) + 1) % 11 == 0
):
# Split into chunks of 10, skipping every 11th character (the separator)
chunks = [candidate[i : i + 10] for i in range(0, len(candidate), 11)]
for chunk in chunks:
corrected, is_valid = attempt_ocr_correction(chunk, identifier_type)
if is_valid and corrected not in seen_values:
results.append((corrected, "OCR_CORRECTED"))
seen_values.add(corrected)
continue
logging.debug(f"OCR candidate for {identifier_type}: {candidate}") logging.debug(f"OCR candidate for {identifier_type}: {candidate}")
corrected, is_valid = attempt_ocr_correction(candidate, identifier_type) corrected, is_valid = attempt_ocr_correction(candidate, identifier_type)
if is_valid and corrected not in seen_values: if is_valid and corrected not in seen_values: