"""
Deduplicate historical complaints created by repeated Excel imports.
The importers (import_historical_complaints / import_2025_complaints_basic)
build reference numbers of the form ``CMP-YYYY-MM-NNNN``. When re-run, the
existing dedup logic appended letter suffixes (-A, -B, ... -H) instead of
skipping, producing duplicate rows with byte-identical content.
A wrinkle: the same ``complaint_num`` (column 3) is occasionally reused across
genuinely distinct complaints, so a single base reference (e.g.
``CMP-2025-10-5112``) can map to several different complaints that were each
re-imported. To handle this safely, dedup is done by
**base reference + content signature**:
1. Group every complaint (base + suffixed) by its stripped base ref.
2. Within each group, cluster by (title, description, created_at).
3. In each cluster, keep ONE representative - preferring the no-suffix base
if it is in that cluster, else the lowest-suffix copy.
4. Delete the rest. ComplaintUpdate rows on the deleted copies are removed
too (they are re-import artifacts).
Usage:
# Dry run (default) - no writes
python manage.py dedup_historical_complaints
python manage.py dedup_historical_complaints --year 2025
python manage.py dedup_historical_complaints --hospital-code HH-N
# Actually delete (requires explicit --confirm)
python manage.py dedup_historical_complaints --confirm
"""
import re
from django.core.management.base import BaseCommand
from django.db import transaction
from apps.complaints.models import Complaint, ComplaintUpdate
from apps.organizations.models import Hospital
# Anchored pattern: CMP-YYYY-MM-NNNN optionally followed by a single uppercase
# letter suffix. Base refs always end in 4 digits, so the suffixed branch is
# never matched on a legitimate base.
REF_RE = re.compile(r"^(?PCMP-\d{4}-\d{2}-\d{4})(?:-(?P[A-Z]))?$")
def _content_signature(c):
return (c.title or "", c.description or "", c.created_at)
def _rank(c):
"""Lower rank = preferred keeper. No-suffix base ranks first (0), then -A (1), -B (2)..."""
m = REF_RE.match(c.reference_number or "")
suffix = m.group("suffix") if m and m.group("suffix") else ""
return (0 if suffix == "" else ord(suffix), str(c.id))
class Command(BaseCommand):
help = "Deduplicate historical complaints that were re-imported with letter suffixes"
def add_arguments(self, parser):
parser.add_argument("--year", type=int, help="Scope to a single year (e.g. 2025)")
parser.add_argument(
"--hospital-code",
type=str,
default="HH-N",
help="Hospital code to scope (default: HH-N)",
)
parser.add_argument(
"--confirm",
action="store_true",
help="Actually perform the deletion. Without it, the command only reports.",
)
def handle(self, *args, **options):
year = options.get("year")
hospital_code = options["hospital_code"]
confirm = options["confirm"]
# Validate hospital (but don't hard-fail if it doesn't exist; just don't filter)
hospital = None
try:
hospital = Hospital.objects.get(code=hospital_code)
except Hospital.DoesNotExist:
self.stdout.write(self.style.WARNING(
f"Hospital code '{hospital_code}' not found - running across all hospitals."
))
qs = Complaint.objects.all()
if hospital:
qs = qs.filter(hospital=hospital)
if year:
qs = qs.filter(created_at__year=year)
# Pull every complaint whose reference matches the base-or-suffixed pattern,
# plus group them by their stripped base ref. We include the no-suffix base
# itself so that content-collision groups are handled correctly.
all_rows = list(
qs.only("id", "reference_number", "created_at", "title", "description")
.filter(reference_number__regex=r"^CMP-\d{4}-\d{2}-\d{4}(?:-[A-Z])?$")
)
from collections import defaultdict
by_base = defaultdict(list)
for c in all_rows:
m = REF_RE.match(c.reference_number or "")
if not m:
continue
by_base[m.group("base")].append(c)
candidate_suffixed = 0
for c in all_rows:
m = REF_RE.match(c.reference_number or "")
if m and m.group("suffix"):
candidate_suffixed += 1
self.stdout.write(self.style.SUCCESS("=" * 70))
if confirm:
self.stdout.write(self.style.SUCCESS("DEDUP - CONFIRM MODE (deletions will run)"))
else:
self.stdout.write(self.style.SUCCESS("DEDUP - DRY RUN (no writes)"))
self.stdout.write(self.style.SUCCESS("=" * 70))
scope = f"hospital={hospital_code}" + (f", year={year}" if year else ", all years")
self.stdout.write(f"Scope: {scope}")
self.stdout.write(f"Complaints matching reference pattern: {len(all_rows)}")
self.stdout.write(f" of which suffixed (-A/-B/...): {candidate_suffixed}")
self.stdout.write(f"Distinct base refs: {len(by_base)}")
self.stdout.write("")
# Content-aware clustering: within each base group, cluster by content.
# Keep one representative per cluster, delete the rest.
to_delete_ids = []
true_unique = 0
pure_dup_groups = 0 # base refs whose rows are all identical content
collision_groups = 0 # base refs with >1 distinct content
per_year_deleted = {}
collisions_detail = []
for base_ref, items in by_base.items():
clusters = defaultdict(list)
for c in items:
clusters[_content_signature(c)].append(c)
if len(clusters) == 1:
pure_dup_groups += 1
else:
collision_groups += 1
collisions_detail.append((base_ref, len(items), len(clusters)))
for _sig, members in clusters.items():
true_unique += 1
members_sorted = sorted(members, key=_rank)
for v in members_sorted[1:]:
y = v.created_at.year if v.created_at else 0
per_year_deleted[y] = per_year_deleted.get(y, 0) + 1
to_delete_ids.append(v.id)
# Count updates attached to the to-be-deleted complaints
updates_to_delete_ids = list(
ComplaintUpdate.objects.filter(
complaint_id__in=to_delete_ids
).values_list("id", flat=True)
) if to_delete_ids else []
# Report
self.stdout.write(self.style.SUCCESS("Plan summary"))
self.stdout.write("-" * 70)
self.stdout.write(f"Base refs - pure re-import dups: {pure_dup_groups}")
self.stdout.write(f"Base refs - content collisions (>1 text): {collision_groups}")
self.stdout.write(f"TRUE unique complaints to keep: {true_unique}")
self.stdout.write(f"Duplicate complaints to delete: {len(to_delete_ids)}")
self.stdout.write(f"ComplaintUpdate rows to delete: {len(updates_to_delete_ids)}")
self.stdout.write("")
self.stdout.write("Per-year complaints to delete:")
for y in sorted(per_year_deleted):
self.stdout.write(f" {y}: {per_year_deleted[y]}")
if collision_groups:
self.stdout.write("")
self.stdout.write(self.style.WARNING(
f"{collision_groups} base refs contain >1 distinct complaint "
f"(reference collision). One copy of EACH distinct text is kept."
))
for base_ref, n_rows, n_clusters in collisions_detail[:15]:
self.stdout.write(f" {base_ref}: {n_rows} rows -> {n_clusters} distinct complaints kept")
if not confirm:
self.stdout.write("")
self.stdout.write(self.style.WARNING("DRY RUN - no changes made. Re-run with --confirm to delete."))
return
# Execute
self.stdout.write("")
self.stdout.write(self.style.SUCCESS("Executing deletion in a single transaction..."))
with transaction.atomic():
deleted_updates, _ = ComplaintUpdate.objects.filter(
id__in=updates_to_delete_ids
).delete()
deleted_complaints, _ = Complaint.objects.filter(
id__in=to_delete_ids
).delete()
self.stdout.write(self.style.SUCCESS("Done."))
self.stdout.write(f"Deleted ComplaintUpdate rows: {deleted_updates}")
self.stdout.write(f"Deleted Complaint rows: {deleted_complaints}")