206 lines
8.5 KiB
Python
206 lines
8.5 KiB
Python
"""
|
|
Deduplicate historical complaints created by repeated Excel imports.
|
|
|
|
The importers (import_historical_complaints / import_2025_complaints_basic)
|
|
build reference numbers of the form ``CMP-YYYY-MM-NNNN``. When re-run, the
|
|
existing dedup logic appended letter suffixes (-A, -B, ... -H) instead of
|
|
skipping, producing duplicate rows with byte-identical content.
|
|
|
|
A wrinkle: the same ``complaint_num`` (column 3) is occasionally reused across
|
|
genuinely distinct complaints, so a single base reference (e.g.
|
|
``CMP-2025-10-5112``) can map to several different complaints that were each
|
|
re-imported. To handle this safely, dedup is done by
|
|
**base reference + content signature**:
|
|
|
|
1. Group every complaint (base + suffixed) by its stripped base ref.
|
|
2. Within each group, cluster by (title, description, created_at).
|
|
3. In each cluster, keep ONE representative - preferring the no-suffix base
|
|
if it is in that cluster, else the lowest-suffix copy.
|
|
4. Delete the rest. ComplaintUpdate rows on the deleted copies are removed
|
|
too (they are re-import artifacts).
|
|
|
|
Usage:
|
|
# Dry run (default) - no writes
|
|
python manage.py dedup_historical_complaints
|
|
python manage.py dedup_historical_complaints --year 2025
|
|
python manage.py dedup_historical_complaints --hospital-code HH-N
|
|
|
|
# Actually delete (requires explicit --confirm)
|
|
python manage.py dedup_historical_complaints --confirm
|
|
"""
|
|
|
|
import re
|
|
|
|
from django.core.management.base import BaseCommand
|
|
from django.db import transaction
|
|
|
|
from apps.complaints.models import Complaint, ComplaintUpdate
|
|
from apps.organizations.models import Hospital
|
|
|
|
# Anchored pattern: CMP-YYYY-MM-NNNN optionally followed by a single uppercase
|
|
# letter suffix. Base refs always end in 4 digits, so the suffixed branch is
|
|
# never matched on a legitimate base.
|
|
REF_RE = re.compile(r"^(?P<base>CMP-\d{4}-\d{2}-\d{4})(?:-(?P<suffix>[A-Z]))?$")
|
|
|
|
|
|
def _content_signature(c):
|
|
return (c.title or "", c.description or "", c.created_at)
|
|
|
|
|
|
def _rank(c):
|
|
"""Lower rank = preferred keeper. No-suffix base ranks first (0), then -A (1), -B (2)..."""
|
|
m = REF_RE.match(c.reference_number or "")
|
|
suffix = m.group("suffix") if m and m.group("suffix") else ""
|
|
return (0 if suffix == "" else ord(suffix), str(c.id))
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Deduplicate historical complaints that were re-imported with letter suffixes"
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("--year", type=int, help="Scope to a single year (e.g. 2025)")
|
|
parser.add_argument(
|
|
"--hospital-code",
|
|
type=str,
|
|
default="HH-N",
|
|
help="Hospital code to scope (default: HH-N)",
|
|
)
|
|
parser.add_argument(
|
|
"--confirm",
|
|
action="store_true",
|
|
help="Actually perform the deletion. Without it, the command only reports.",
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
year = options.get("year")
|
|
hospital_code = options["hospital_code"]
|
|
confirm = options["confirm"]
|
|
|
|
# Validate hospital (but don't hard-fail if it doesn't exist; just don't filter)
|
|
hospital = None
|
|
try:
|
|
hospital = Hospital.objects.get(code=hospital_code)
|
|
except Hospital.DoesNotExist:
|
|
self.stdout.write(self.style.WARNING(
|
|
f"Hospital code '{hospital_code}' not found - running across all hospitals."
|
|
))
|
|
|
|
qs = Complaint.objects.all()
|
|
if hospital:
|
|
qs = qs.filter(hospital=hospital)
|
|
if year:
|
|
qs = qs.filter(created_at__year=year)
|
|
|
|
# Pull every complaint whose reference matches the base-or-suffixed pattern,
|
|
# plus group them by their stripped base ref. We include the no-suffix base
|
|
# itself so that content-collision groups are handled correctly.
|
|
all_rows = list(
|
|
qs.only("id", "reference_number", "created_at", "title", "description")
|
|
.filter(reference_number__regex=r"^CMP-\d{4}-\d{2}-\d{4}(?:-[A-Z])?$")
|
|
)
|
|
|
|
from collections import defaultdict
|
|
|
|
by_base = defaultdict(list)
|
|
for c in all_rows:
|
|
m = REF_RE.match(c.reference_number or "")
|
|
if not m:
|
|
continue
|
|
by_base[m.group("base")].append(c)
|
|
|
|
candidate_suffixed = 0
|
|
for c in all_rows:
|
|
m = REF_RE.match(c.reference_number or "")
|
|
if m and m.group("suffix"):
|
|
candidate_suffixed += 1
|
|
|
|
self.stdout.write(self.style.SUCCESS("=" * 70))
|
|
if confirm:
|
|
self.stdout.write(self.style.SUCCESS("DEDUP - CONFIRM MODE (deletions will run)"))
|
|
else:
|
|
self.stdout.write(self.style.SUCCESS("DEDUP - DRY RUN (no writes)"))
|
|
self.stdout.write(self.style.SUCCESS("=" * 70))
|
|
scope = f"hospital={hospital_code}" + (f", year={year}" if year else ", all years")
|
|
self.stdout.write(f"Scope: {scope}")
|
|
self.stdout.write(f"Complaints matching reference pattern: {len(all_rows)}")
|
|
self.stdout.write(f" of which suffixed (-A/-B/...): {candidate_suffixed}")
|
|
self.stdout.write(f"Distinct base refs: {len(by_base)}")
|
|
self.stdout.write("")
|
|
|
|
# Content-aware clustering: within each base group, cluster by content.
|
|
# Keep one representative per cluster, delete the rest.
|
|
to_delete_ids = []
|
|
true_unique = 0
|
|
pure_dup_groups = 0 # base refs whose rows are all identical content
|
|
collision_groups = 0 # base refs with >1 distinct content
|
|
per_year_deleted = {}
|
|
collisions_detail = []
|
|
|
|
for base_ref, items in by_base.items():
|
|
clusters = defaultdict(list)
|
|
for c in items:
|
|
clusters[_content_signature(c)].append(c)
|
|
|
|
if len(clusters) == 1:
|
|
pure_dup_groups += 1
|
|
else:
|
|
collision_groups += 1
|
|
collisions_detail.append((base_ref, len(items), len(clusters)))
|
|
|
|
for _sig, members in clusters.items():
|
|
true_unique += 1
|
|
members_sorted = sorted(members, key=_rank)
|
|
for v in members_sorted[1:]:
|
|
y = v.created_at.year if v.created_at else 0
|
|
per_year_deleted[y] = per_year_deleted.get(y, 0) + 1
|
|
to_delete_ids.append(v.id)
|
|
|
|
# Count updates attached to the to-be-deleted complaints
|
|
updates_to_delete_ids = list(
|
|
ComplaintUpdate.objects.filter(
|
|
complaint_id__in=to_delete_ids
|
|
).values_list("id", flat=True)
|
|
) if to_delete_ids else []
|
|
|
|
# Report
|
|
self.stdout.write(self.style.SUCCESS("Plan summary"))
|
|
self.stdout.write("-" * 70)
|
|
self.stdout.write(f"Base refs - pure re-import dups: {pure_dup_groups}")
|
|
self.stdout.write(f"Base refs - content collisions (>1 text): {collision_groups}")
|
|
self.stdout.write(f"TRUE unique complaints to keep: {true_unique}")
|
|
self.stdout.write(f"Duplicate complaints to delete: {len(to_delete_ids)}")
|
|
self.stdout.write(f"ComplaintUpdate rows to delete: {len(updates_to_delete_ids)}")
|
|
self.stdout.write("")
|
|
self.stdout.write("Per-year complaints to delete:")
|
|
for y in sorted(per_year_deleted):
|
|
self.stdout.write(f" {y}: {per_year_deleted[y]}")
|
|
|
|
if collision_groups:
|
|
self.stdout.write("")
|
|
self.stdout.write(self.style.WARNING(
|
|
f"{collision_groups} base refs contain >1 distinct complaint "
|
|
f"(reference collision). One copy of EACH distinct text is kept."
|
|
))
|
|
for base_ref, n_rows, n_clusters in collisions_detail[:15]:
|
|
self.stdout.write(f" {base_ref}: {n_rows} rows -> {n_clusters} distinct complaints kept")
|
|
|
|
if not confirm:
|
|
self.stdout.write("")
|
|
self.stdout.write(self.style.WARNING("DRY RUN - no changes made. Re-run with --confirm to delete."))
|
|
return
|
|
|
|
# Execute
|
|
self.stdout.write("")
|
|
self.stdout.write(self.style.SUCCESS("Executing deletion in a single transaction..."))
|
|
with transaction.atomic():
|
|
deleted_updates, _ = ComplaintUpdate.objects.filter(
|
|
id__in=updates_to_delete_ids
|
|
).delete()
|
|
deleted_complaints, _ = Complaint.objects.filter(
|
|
id__in=to_delete_ids
|
|
).delete()
|
|
|
|
self.stdout.write(self.style.SUCCESS("Done."))
|
|
self.stdout.write(f"Deleted ComplaintUpdate rows: {deleted_updates}")
|
|
self.stdout.write(f"Deleted Complaint rows: {deleted_complaints}")
|