319 lines
13 KiB
Python
319 lines
13 KiB
Python
# integration/management/commands/generate_integration_data.py
|
||
|
||
import random
|
||
import uuid
|
||
from datetime import datetime, timedelta
|
||
|
||
from django.core.management.base import BaseCommand
|
||
from django.utils import timezone
|
||
from django.contrib.auth import get_user_model
|
||
|
||
from faker import Faker
|
||
|
||
from integration.models import (
|
||
ExternalSystem, IntegrationEndpoint,
|
||
DataMapping, IntegrationExecution,
|
||
WebhookEndpoint, WebhookExecution,
|
||
IntegrationLog
|
||
)
|
||
from core.models import Tenant
|
||
|
||
User = get_user_model()
|
||
fake = Faker('en_US')
|
||
|
||
|
||
class Command(BaseCommand):
|
||
help = "Generate sample data for the integration app"
|
||
|
||
# ------------------------------------------------------------------
|
||
# Utility helpers
|
||
# ------------------------------------------------------------------
|
||
@staticmethod
|
||
def random_choice(field, exclude=None):
|
||
"""Return a random value from a Django field’s choices."""
|
||
choices = field.choices
|
||
if exclude:
|
||
choices = [c for c in choices if c[0] not in exclude]
|
||
return random.choice(choices)[0]
|
||
|
||
@staticmethod
|
||
def random_bool(chance=0.5):
|
||
return random.random() < chance
|
||
|
||
# ------------------------------------------------------------------
|
||
# Main entry point
|
||
# ------------------------------------------------------------------
|
||
def handle(self, *args, **options):
|
||
tenants = Tenant.objects.all()
|
||
if not tenants:
|
||
self.stdout.write(self.style.ERROR("No tenants found. Create tenants before generating data."))
|
||
return
|
||
|
||
for tenant in tenants:
|
||
self.generate_external_systems(tenant)
|
||
self.generate_webhooks(tenant)
|
||
|
||
self.stdout.write(self.style.SUCCESS("Integration data generation complete."))
|
||
|
||
# ------------------------------------------------------------------
|
||
# External system + endpoints
|
||
# ------------------------------------------------------------------
|
||
def generate_external_systems(self, tenant, count=5):
|
||
for _ in range(count):
|
||
system = ExternalSystem.objects.create(
|
||
tenant=tenant,
|
||
name=fake.company() + " " + self.random_choice(ExternalSystem.SYSTEM_TYPES,
|
||
exclude=['other']),
|
||
description=fake.text(max_nb_chars=200),
|
||
system_type=self.random_choice(ExternalSystem.SYSTEM_TYPES),
|
||
vendor=fake.company(),
|
||
version=f"{random.randint(1, 5)}.{random.randint(0, 9)}",
|
||
base_url=fake.url(),
|
||
host=fake.hostname(),
|
||
port=random.randint(1024, 65535),
|
||
authentication_type=self.random_choice(ExternalSystem.AUTHENTICATION_TYPES),
|
||
authentication_config=self._random_auth_config(),
|
||
configuration=self._random_config(),
|
||
timeout_seconds=random.randint(10, 60),
|
||
retry_attempts=random.randint(0, 5),
|
||
retry_delay_seconds=random.randint(1, 10),
|
||
is_active=self.random_bool(0.9),
|
||
is_healthy=self.random_bool(0.7),
|
||
health_check_interval=random.randint(120, 600),
|
||
connection_count=random.randint(0, 1000),
|
||
success_count=random.randint(0, 800),
|
||
failure_count=random.randint(0, 200),
|
||
last_used_at=timezone.now() - timedelta(days=random.randint(0, 30)),
|
||
created_by=User.objects.filter(is_staff=True).first()
|
||
)
|
||
self.generate_endpoints(system, count=3)
|
||
|
||
def generate_endpoints(self, system, count=3):
|
||
for _ in range(count):
|
||
endpoint = IntegrationEndpoint.objects.create(
|
||
external_system=system,
|
||
name=fake.word().title() + " API",
|
||
description=fake.text(max_nb_chars=150),
|
||
endpoint_type=self.random_choice(IntegrationEndpoint.ENDPOINT_TYPES),
|
||
path=f"/{fake.word()}/{fake.word()}",
|
||
method=self.random_choice(IntegrationEndpoint.METHODS),
|
||
direction=self.random_choice(IntegrationEndpoint.DIRECTIONS),
|
||
headers=self._random_headers(),
|
||
parameters=self._random_params(),
|
||
request_format="json",
|
||
response_format="json",
|
||
request_mapping=self._random_mapping(),
|
||
response_mapping=self._random_mapping(),
|
||
request_schema=self._random_schema(),
|
||
response_schema=self._random_schema(),
|
||
is_active=self.random_bool(0.9),
|
||
created_by=User.objects.filter(is_staff=True).first()
|
||
)
|
||
# Data mappings belong to an endpoint – create a few of them
|
||
for _ in range(random.randint(0, 2)):
|
||
DataMapping.objects.create(
|
||
endpoint=endpoint,
|
||
name=fake.word().title() + " Mapping",
|
||
description=fake.text(max_nb_chars=100),
|
||
mapping_type=self.random_choice(DataMapping.MAPPING_TYPES),
|
||
source_field=fake.word(),
|
||
source_format="json",
|
||
source_validation=self._random_validation(),
|
||
target_field=fake.word(),
|
||
target_format="json",
|
||
target_validation=self._random_validation(),
|
||
transformation_type=self.random_choice(DataMapping.TRANSFORMATION_TYPES),
|
||
transformation_config=self._random_config(),
|
||
is_required=self.random_bool(),
|
||
validation_rules=self._random_validation(),
|
||
default_value=fake.word(),
|
||
is_active=self.random_bool(),
|
||
created_by=User.objects.filter(is_staff=True).first()
|
||
)
|
||
self.generate_executions(endpoint, count=random.randint(0, 5))
|
||
|
||
# ------------------------------------------------------------------
|
||
# Execution history
|
||
# ------------------------------------------------------------------
|
||
def generate_executions(self, endpoint, count=3):
|
||
for _ in range(count):
|
||
started = timezone.now() - timedelta(minutes=random.randint(0, 120))
|
||
completed = started + timedelta(seconds=random.randint(10, 120))
|
||
if self.random_bool(0.1):
|
||
completed = None # simulate in‑flight or stuck
|
||
|
||
exec = IntegrationExecution.objects.create(
|
||
endpoint=endpoint,
|
||
execution_type=self.random_choice(IntegrationExecution.EXECUTION_TYPES),
|
||
status=self.random_choice(IntegrationExecution.STATUSES),
|
||
started_at=started,
|
||
completed_at=completed,
|
||
request_data=self._random_json(),
|
||
response_data=self._random_json(),
|
||
request_size_bytes=random.randint(100, 2000),
|
||
response_size_bytes=random.randint(100, 2000),
|
||
processing_time_ms=random.randint(50, 2000),
|
||
network_time_ms=random.randint(10, 500),
|
||
error_message="" if self.random_bool(0.9) else fake.sentence(),
|
||
error_details=self._random_json() if self.random_bool(0.2) else {},
|
||
retry_count=random.randint(0, 3),
|
||
external_id=fake.uuid4(),
|
||
correlation_id=fake.uuid4(),
|
||
triggered_by=User.objects.filter(is_staff=True).first(),
|
||
metadata=self._random_json()
|
||
)
|
||
|
||
# Webhook executions (if the endpoint is a webhook)
|
||
if endpoint.endpoint_type == "webhook":
|
||
self.generate_webhook_executions(endpoint.external_system, exec)
|
||
|
||
# Logs – one per execution
|
||
self.generate_logs(exec, count=random.randint(1, 4))
|
||
|
||
# ------------------------------------------------------------------
|
||
# Webhook endpoints & executions
|
||
# ------------------------------------------------------------------
|
||
def generate_webhooks(self, tenant, count=2):
|
||
for _ in range(count):
|
||
system = ExternalSystem.objects.filter(tenant=tenant).first()
|
||
if not system:
|
||
continue
|
||
|
||
webhook = WebhookEndpoint.objects.create(
|
||
external_system=system,
|
||
name=fake.word().title() + " Webhook",
|
||
description=fake.text(max_nb_chars=120),
|
||
url_path=f"/webhooks/{fake.slug()}",
|
||
allowed_methods=[self.random_choice(IntegrationEndpoint.METHODS)],
|
||
authentication_type=self.random_choice(WebhookEndpoint.AUTHENTICATION_TYPES),
|
||
authentication_config=self._random_auth_config(),
|
||
processing_config=self._random_config(),
|
||
rate_limit_per_minute=random.randint(30, 120),
|
||
rate_limit_per_hour=random.randint(500, 2000),
|
||
is_active=self.random_bool(0.9),
|
||
created_by=User.objects.filter(is_staff=True).first()
|
||
)
|
||
|
||
# Optionally link a data mapping
|
||
if DataMapping.objects.exists():
|
||
webhook.data_mapping = random.choice(list(DataMapping.objects.all()))
|
||
webhook.save()
|
||
|
||
self.generate_webhook_executions(webhook, None)
|
||
|
||
# Logs for the webhook itself
|
||
self.generate_logs(webhook, count=random.randint(1, 3))
|
||
|
||
def generate_webhook_executions(self, webhook, parent_exec=None):
|
||
for _ in range(random.randint(0, 3)):
|
||
method = self.random_choice(IntegrationEndpoint.METHODS)
|
||
status = self.random_choice(WebhookExecution.STATUSES)
|
||
received_at = timezone.now() - timedelta(seconds=random.randint(0, 300))
|
||
processed_at = received_at + timedelta(seconds=random.randint(0, 30)) if status in ("completed", "failed") else None
|
||
|
||
WebhookExecution.objects.create(
|
||
webhook=webhook,
|
||
method=method,
|
||
headers=self._random_headers(),
|
||
query_params=self._random_params(),
|
||
payload=self._random_json(),
|
||
payload_size_bytes=random.randint(100, 2000),
|
||
client_ip=fake.ipv4_public(),
|
||
user_agent=fake.user_agent(),
|
||
status=status,
|
||
received_at=received_at,
|
||
processed_at=processed_at,
|
||
processing_time_ms=random.randint(50, 2000) if processed_at else None,
|
||
response_status=200 if status == "completed" else random.choice([400, 401, 403, 500]),
|
||
response_data=self._random_json(),
|
||
error_message="" if status == "completed" else fake.sentence(),
|
||
error_details=self._random_json() if status != "completed" else {},
|
||
external_id=fake.uuid4(),
|
||
correlation_id=fake.uuid4(),
|
||
metadata=self._random_json()
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Integration logs (generic)
|
||
# ------------------------------------------------------------------
|
||
def generate_logs(self, source, count=5):
|
||
for _ in range(count):
|
||
timestamp = timezone.now() - timedelta(minutes=random.randint(0, 120))
|
||
log = IntegrationLog.objects.create(
|
||
external_system=getattr(source, "external_system", None),
|
||
endpoint=getattr(source, "endpoint", None),
|
||
execution=getattr(source, "execution", None),
|
||
level=self.random_choice(IntegrationLog.LOG_LEVELS),
|
||
category=self.random_choice(IntegrationLog.CATEGORIES),
|
||
message=fake.sentence(),
|
||
details=self._random_json(),
|
||
correlation_id=fake.uuid4(),
|
||
user=User.objects.filter(is_staff=True).first(),
|
||
timestamp=timestamp,
|
||
metadata=self._random_json()
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Random helpers
|
||
# ------------------------------------------------------------------
|
||
@staticmethod
|
||
def _random_json():
|
||
return {
|
||
"key1": fake.word(),
|
||
"key2": fake.word(),
|
||
"nested": {"sub": fake.word()}
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_headers():
|
||
return {
|
||
"Authorization": f"Bearer {uuid.uuid4()}",
|
||
"Content-Type": "application/json"
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_params():
|
||
return {
|
||
fake.word(): fake.word(),
|
||
fake.word(): fake.word()
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_mapping():
|
||
return {
|
||
"source": fake.word(),
|
||
"target": fake.word()
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_schema():
|
||
return {
|
||
"type": "object",
|
||
"properties": {
|
||
fake.word(): {"type": "string"},
|
||
fake.word(): {"type": "integer"}
|
||
}
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_validation():
|
||
return {
|
||
"regex": "^[A-Za-z0-9_]+$",
|
||
"max_length": 50
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_auth_config():
|
||
return {
|
||
"username": fake.user_name(),
|
||
"password": fake.password(),
|
||
"token_url": fake.url()
|
||
}
|
||
|
||
@staticmethod
|
||
def _random_config():
|
||
return {
|
||
"env": random.choice(["dev", "test", "prod"]),
|
||
"retries": random.randint(0, 5),
|
||
"timeout": random.randint(10, 60)
|
||
} |