362 lines
14 KiB
Python
362 lines
14 KiB
Python
"""
|
|
Import 2025 complaints from Excel with basic fields (no AI, skip missing columns).
|
|
|
|
Dynamically detects header row and column positions per sheet because 2025
|
|
monthly sheets have inconsistent layouts.
|
|
|
|
Usage:
|
|
python manage.py import_2025_complaints_basic "Complaints Report - 2025.xlsx" --sheet="JAN"
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Optional
|
|
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.accounts.models import User
|
|
from apps.complaints.models import Complaint
|
|
from apps.organizations.models import Hospital, Location, MainSection, SubSection
|
|
|
|
from .complaint_source_mapping import resolve_px_source
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_HOSPITAL_CODE = "NUZHA"
|
|
|
|
# Header aliases: list of possible names in Excel for each field
|
|
HEADER_ALIASES = {
|
|
"complaint_num": ["رقم الشكوى"],
|
|
"mrn": ["رقم الملف"],
|
|
"source": ["جهة الشكوى"],
|
|
"location_name": ["الموقع"],
|
|
"main_dept_name": ["القسم الرئيس"],
|
|
"sub_dept_name": ["القسم الفرعي"],
|
|
"date_received": ["تاريخ إستلام الشكوى"],
|
|
"data_entry_person": ["المدخل"],
|
|
"response_date": ["تاريخ الرد"],
|
|
"staff_name": ["اسم الشخص المشتكى عليه - ان وجد", "اسم الشخص المشتكى عليه"],
|
|
"description_ar": ["الشكوى باختصار (عربي)", "محتوى الشكوى (عربي)"],
|
|
"description_en": ["الشكوى باختصار English", "محتوى الشكوى (English)"],
|
|
"satisfaction": ["توثيق تذكيرات للقسم المشتكى عليه"],
|
|
"reminder_date": ["تاريخ التذكير"],
|
|
}
|
|
|
|
MONTH_MAP = {
|
|
"JAN": "01",
|
|
"FEB": "02",
|
|
"MAR": "03",
|
|
"APR": "04",
|
|
"MAY": "05",
|
|
"JUN": "06",
|
|
"JUL": "07",
|
|
"AUG": "08",
|
|
"SEP": "09",
|
|
"OCT": "10",
|
|
"NOV": "11",
|
|
"DEC": "12",
|
|
}
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Import 2025 complaints with basic fields (no taxonomy, no staff linking)"
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("excel_file", type=str)
|
|
parser.add_argument("--sheet", type=str, default="JAN")
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
|
|
def handle(self, *args, **options):
|
|
self.excel_file = options["excel_file"]
|
|
self.sheet_name = options["sheet"]
|
|
self.dry_run = options["dry_run"]
|
|
|
|
self.hospital = self._load_hospital()
|
|
if not self.hospital:
|
|
raise CommandError(f'Hospital "{DEFAULT_HOSPITAL_CODE}" not found')
|
|
|
|
self.stdout.write(f"Using hospital: {self.hospital.name}")
|
|
|
|
try:
|
|
import openpyxl
|
|
|
|
self.wb = openpyxl.load_workbook(self.excel_file, read_only=True, data_only=True)
|
|
except ImportError:
|
|
raise CommandError("openpyxl required: pip install openpyxl")
|
|
|
|
if self.sheet_name not in self.wb.sheetnames:
|
|
available = ", ".join(self.wb.sheetnames)
|
|
raise CommandError(f'Sheet "{self.sheet_name}" not found. Available: {available}')
|
|
|
|
self.ws = self.wb[self.sheet_name]
|
|
self.stdout.write(f"Processing sheet: {self.sheet_name}")
|
|
|
|
# Detect header row and build column mapping
|
|
self.column_map = self._detect_columns()
|
|
if not self.column_map.get("complaint_num"):
|
|
raise CommandError("Could not detect 'رقم الشكوى' header in sheet")
|
|
|
|
self.stdout.write(f"Detected columns: {self.column_map}")
|
|
|
|
self.stats = {"processed": 0, "success": 0, "failed": 0}
|
|
self.errors = []
|
|
self.used_refs = set()
|
|
|
|
self._process_sheet()
|
|
self._print_report()
|
|
|
|
def _load_hospital(self) -> Optional[Hospital]:
|
|
try:
|
|
return Hospital.objects.get(code=DEFAULT_HOSPITAL_CODE)
|
|
except Hospital.DoesNotExist:
|
|
return None
|
|
|
|
def _detect_columns(self) -> Dict[str, int]:
|
|
"""Scan first 10 rows to find header row and map columns."""
|
|
mapping = {}
|
|
for r in range(1, 11):
|
|
row_values = {}
|
|
for c in range(1, 80):
|
|
val = self.ws.cell(r, c).value
|
|
if val:
|
|
# Keep first occurrence only
|
|
if val not in row_values:
|
|
row_values[val] = c
|
|
# Check if this row contains the complaint number header
|
|
if "رقم الشكوى" in row_values:
|
|
for field, aliases in HEADER_ALIASES.items():
|
|
for alias in aliases:
|
|
if alias in row_values:
|
|
mapping[field] = row_values[alias]
|
|
break
|
|
self.header_row = r
|
|
break
|
|
return mapping
|
|
|
|
def _process_sheet(self):
|
|
row_num = self.header_row + 1
|
|
|
|
for row in self.ws.iter_rows(min_row=self.header_row + 1, max_row=5000, values_only=True):
|
|
try:
|
|
row_data = self._extract_row_data_from_values(row)
|
|
|
|
if not row_data.get("complaint_num"):
|
|
row_num += 1
|
|
continue
|
|
|
|
self.stats["processed"] += 1
|
|
|
|
try:
|
|
ref_num = self._get_unique_reference_number(row_data["complaint_num"])
|
|
except (ValueError, TypeError):
|
|
row_num += 1
|
|
continue
|
|
|
|
px_source = resolve_px_source(row_data.get("source"))
|
|
|
|
date_received = self._parse_datetime(row_data.get("date_received"))
|
|
created_at = date_received or timezone.now()
|
|
if created_at and timezone.is_naive(created_at):
|
|
created_at = timezone.make_aware(created_at)
|
|
response_date = self._parse_datetime(row_data.get("response_date"))
|
|
reminder_date = self._parse_datetime(row_data.get("reminder_date"))
|
|
|
|
location = self._resolve_location(row_data.get("location_name"))
|
|
main_section = self._resolve_section(row_data.get("main_dept_name"))
|
|
subsection = self._resolve_subsection(row_data.get("sub_dept_name"))
|
|
|
|
assigned_to_user = self._get_or_create_data_entry_user(row_data.get("data_entry_person"))
|
|
|
|
status = "open"
|
|
if response_date:
|
|
status = "resolved"
|
|
|
|
if not self.dry_run:
|
|
with transaction.atomic():
|
|
complaint = Complaint.objects.create(
|
|
reference_number=ref_num,
|
|
hospital=self.hospital,
|
|
location=location,
|
|
main_section=main_section,
|
|
subsection=subsection,
|
|
title=self._build_title(row_data),
|
|
description=self._build_description(row_data),
|
|
patient_name="Unknown",
|
|
national_id="",
|
|
relation_to_patient="patient",
|
|
staff=None,
|
|
staff_name=row_data.get("staff_name") or "",
|
|
domain=None,
|
|
category=None,
|
|
subcategory_obj=None,
|
|
classification_obj=None,
|
|
status=status,
|
|
assigned_to=assigned_to_user,
|
|
resolved_by=assigned_to_user if response_date else None,
|
|
due_at=created_at + timedelta(hours=48),
|
|
explanation_requested=bool(date_received),
|
|
explanation_requested_at=date_received,
|
|
explanation_received_at=response_date,
|
|
reminder_sent_at=reminder_date,
|
|
source=px_source,
|
|
metadata={
|
|
"import_source": "2025_excel_basic",
|
|
"original_sheet": self.sheet_name,
|
|
"complaint_num": row_data.get("complaint_num"),
|
|
},
|
|
)
|
|
Complaint.objects.filter(pk=complaint.pk).update(created_at=created_at)
|
|
|
|
self.stats["success"] += 1
|
|
|
|
except Exception as e:
|
|
self.stats["failed"] += 1
|
|
self.errors.append({"row": row_num, "error": str(e)})
|
|
logger.error(f"Row {row_num}: {e}", exc_info=True)
|
|
|
|
row_num += 1
|
|
|
|
def _extract_row_data(self, row_num: int) -> Dict:
|
|
data = {}
|
|
for field, col in self.column_map.items():
|
|
cell_value = self.ws.cell(row_num, col).value
|
|
data[field] = cell_value
|
|
return data
|
|
|
|
def _extract_row_data_from_values(self, row: tuple) -> Dict:
|
|
data = {}
|
|
for field, col in self.column_map.items():
|
|
cell_value = row[col - 1] if col - 1 < len(row) else None
|
|
data[field] = cell_value
|
|
return data
|
|
|
|
def _build_reference_number(self, complaint_num) -> str:
|
|
sheet_parts = self.sheet_name.strip().split()
|
|
year = "2025"
|
|
month_part = sheet_parts[0].upper()
|
|
month_code = MONTH_MAP.get(month_part, "00")
|
|
return f"CMP-{year}-{month_code}-{int(complaint_num):04d}"
|
|
|
|
def _get_unique_reference_number(self, complaint_num) -> str:
|
|
"""Generate unique reference number with suffix if needed."""
|
|
base_ref = self._build_reference_number(complaint_num)
|
|
|
|
if base_ref not in self.used_refs and not Complaint.objects.filter(reference_number=base_ref).exists():
|
|
self.used_refs.add(base_ref)
|
|
return base_ref
|
|
|
|
suffix = ord("A")
|
|
while suffix <= ord("Z"):
|
|
ref_with_suffix = f"{base_ref}-{chr(suffix)}"
|
|
if (
|
|
ref_with_suffix not in self.used_refs
|
|
and not Complaint.objects.filter(reference_number=ref_with_suffix).exists()
|
|
):
|
|
self.used_refs.add(ref_with_suffix)
|
|
return ref_with_suffix
|
|
suffix += 1
|
|
|
|
raise ValueError(f"Cannot generate unique reference for {base_ref}")
|
|
|
|
def _parse_datetime(self, value) -> Optional[datetime]:
|
|
if not value:
|
|
return None
|
|
if isinstance(value, datetime):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
return datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
|
|
except ValueError:
|
|
try:
|
|
return datetime.strptime(value, "%Y-%m-%d")
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
def _resolve_location(self, name_ar: str) -> Optional[Location]:
|
|
if not name_ar:
|
|
return None
|
|
return Location.objects.filter(name_ar=name_ar).first()
|
|
|
|
def _resolve_section(self, name_ar: str) -> Optional[MainSection]:
|
|
if not name_ar:
|
|
return None
|
|
return MainSection.objects.filter(name_ar=name_ar).first()
|
|
|
|
def _resolve_subsection(self, name_ar: str) -> Optional[SubSection]:
|
|
if not name_ar:
|
|
return None
|
|
return SubSection.objects.filter(name_ar=name_ar).first()
|
|
|
|
def _get_or_create_data_entry_user(self, arabic_name: str) -> Optional[User]:
|
|
if not arabic_name:
|
|
return None
|
|
|
|
try:
|
|
from unidecode import unidecode
|
|
except ImportError:
|
|
return None
|
|
|
|
parts = arabic_name.split()
|
|
if len(parts) >= 2:
|
|
first, last = parts[0], parts[-1]
|
|
else:
|
|
first, last = arabic_name, "staff"
|
|
|
|
username_first = re.sub(r"[^a-z0-9]", "", unidecode(first).lower().strip())
|
|
username_last = re.sub(r"[^a-z0-9]", "", unidecode(last).lower().strip())
|
|
|
|
if not username_first:
|
|
username_first = "user"
|
|
if not username_last:
|
|
username_last = "staff"
|
|
|
|
username = f"{username_first}.{username_last}"
|
|
|
|
user = User.objects.filter(username=username).first()
|
|
if user:
|
|
return user
|
|
|
|
try:
|
|
user = User(
|
|
username=username,
|
|
first_name=arabic_name,
|
|
last_name="",
|
|
email=f"{username}@alhammadi.med.sa",
|
|
is_active=True,
|
|
)
|
|
user.save()
|
|
return user
|
|
except Exception:
|
|
return None
|
|
|
|
def _build_title(self, row_data: Dict) -> str:
|
|
desc = row_data.get("description_en") or row_data.get("description_ar") or ""
|
|
return desc[:500] if desc else "No description"
|
|
|
|
def _build_description(self, row_data: Dict) -> str:
|
|
desc_en = row_data.get("description_en") or ""
|
|
desc_ar = row_data.get("description_ar") or ""
|
|
|
|
if desc_en and desc_ar:
|
|
return f"{desc_en}\n\n[Arabic]:\n{desc_ar}"
|
|
return desc_en or desc_ar or "No description provided"
|
|
|
|
def _print_report(self):
|
|
self.stdout.write("\n" + "=" * 60)
|
|
self.stdout.write("IMPORT REPORT - 2025 BASIC")
|
|
self.stdout.write("=" * 60)
|
|
self.stdout.write(f"Sheet: {self.sheet_name}")
|
|
self.stdout.write(f"Mode: {'DRY RUN' if self.dry_run else 'ACTUAL'}")
|
|
self.stdout.write(f"\nProcessed: {self.stats['processed']}")
|
|
self.stdout.write(f"Success: {self.stats['success']}")
|
|
self.stdout.write(f"Failed: {self.stats['failed']}")
|
|
|
|
if self.errors:
|
|
self.stdout.write(f"\nErrors: {len(self.errors)}")
|
|
for error in self.errors[:5]:
|
|
self.stdout.write(f" Row {error['row']}: {error['error']}")
|