104 lines
4.5 KiB
Python
104 lines
4.5 KiB
Python
import os
|
|
import json
|
|
import pymysql
|
|
import pandas as pd
|
|
from django.core.management.base import BaseCommand
|
|
from sqlalchemy import create_engine
|
|
from tqdm import tqdm # Progress bar support
|
|
from django.conf import settings
|
|
|
|
# Database connection details
|
|
db_config = {
|
|
'host': 'localhost',
|
|
'user': 'root',
|
|
'password': settings.PASSWORD,
|
|
'database': 'car2db01022025'
|
|
}
|
|
|
|
EXCLUDED_TABLES = {"car_serie", "car_generation"} # Tables to exclude from direct dump
|
|
|
|
class Command(BaseCommand):
|
|
help = "Merge car_serie with car_generation, include in final JSON dump with a progress bar."
|
|
|
|
def handle(self, *args, **kwargs):
|
|
try:
|
|
self.stdout.write(self.style.SUCCESS("Connecting to database..."))
|
|
# Create SQLAlchemy engine
|
|
engine = create_engine(f"mysql+pymysql://{db_config['user']}:{db_config['password']}@{db_config['host']}/{db_config['database']}")
|
|
|
|
# Load car_generation table
|
|
self.stdout.write(self.style.SUCCESS("Loading car_generation data..."))
|
|
car_generation_query = "SELECT * FROM car_generation;"
|
|
car_generation_df = pd.read_sql(car_generation_query, engine)
|
|
|
|
# Load car_serie table
|
|
self.stdout.write(self.style.SUCCESS("Loading car_serie data..."))
|
|
car_serie_query = "SELECT * FROM car_serie;"
|
|
car_serie_df = pd.read_sql(car_serie_query, engine)
|
|
|
|
# Perform a LEFT JOIN to keep all car series and merge with car generations
|
|
self.stdout.write(self.style.SUCCESS("Merging car_serie with car_generation..."))
|
|
merged_df = pd.merge(car_serie_df, car_generation_df, on="id_car_generation", how="left")
|
|
|
|
# Select and rename the relevant columns
|
|
final_df = merged_df.rename(columns={
|
|
"id_car_serie": "id_car_serie",
|
|
"id_car_model_x": "id_car_model",
|
|
"name_y": "generation_name",
|
|
"name_x": "serie_name",
|
|
"year_begin": "year_begin",
|
|
"year_end": "year_end"
|
|
})[["id_car_serie", "id_car_model", "generation_name", "serie_name", "year_begin", "year_end"]]
|
|
|
|
# Convert merged data to a JSON-ready format
|
|
self.stdout.write(self.style.SUCCESS("Processing merged data..."))
|
|
car_serie_json = list(tqdm(final_df.to_dict(orient="records"), desc="Processing car_serie"))
|
|
|
|
# Export the full database including merged car_serie
|
|
self.export_database_to_json(car_serie_json)
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.ERROR(f"Error: {e}"))
|
|
|
|
def export_database_to_json(self, car_serie_data):
|
|
""" Export the entire MariaDB database to JSON, replacing car_serie with merged data """
|
|
try:
|
|
self.stdout.write(self.style.SUCCESS("Exporting database to JSON..."))
|
|
# Connect to the database using pymysql
|
|
connection = pymysql.connect(**db_config)
|
|
cursor = connection.cursor()
|
|
|
|
# Fetch all table names
|
|
cursor.execute("SHOW TABLES")
|
|
tables = cursor.fetchall()
|
|
|
|
# Function to fetch data from a table and convert it to JSON
|
|
def table_to_json(table_name):
|
|
table_cursor = connection.cursor(pymysql.cursors.DictCursor)
|
|
table_cursor.execute(f"SELECT * FROM {table_name}")
|
|
rows = table_cursor.fetchall()
|
|
return rows
|
|
|
|
# Export each table to JSON, excluding `car_serie` and `car_generation`
|
|
database_json = {}
|
|
self.stdout.write(self.style.SUCCESS("Processing database tables..."))
|
|
for table in tqdm(tables, desc="Exporting tables"):
|
|
table_name = table[0]
|
|
if table_name not in EXCLUDED_TABLES:
|
|
database_json[table_name] = table_to_json(table_name)
|
|
|
|
# Add the merged car_serie data to the final export
|
|
database_json["car_serie"] = car_serie_data
|
|
|
|
# Save the JSON to a file
|
|
self.stdout.write(self.style.SUCCESS("Saving database_export.json..."))
|
|
with open('database_export.json', 'w', encoding='utf-8') as json_file:
|
|
json.dump(database_json, json_file, indent=4, ensure_ascii=False)
|
|
|
|
self.stdout.write(self.style.SUCCESS("✅ Database exported to JSON successfully! (Including merged car_serie)"))
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.ERROR(f"Error exporting database: {e}"))
|
|
finally:
|
|
if connection:
|
|
connection.close() |