Exploratory Data Analysis (EDA) — Complete Guide

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Load the dataset
df = pd.read_csv('india_ecommerce_orders.csv', parse_dates=['order_date'])

# --- Basic structure ---
print(f"Rows: {df.shape[0]:,}  |  Columns: {df.shape[1]}")
print(f"Date range: {df['order_date'].min().date()} → {df['order_date'].max().date()}")

# Column names and types
print(df.dtypes)

# First 5 rows
print(df.head())

# Summary of all columns at once
print(df.info())
# info() shows: column name, count of non-null values, data type
# Immediately reveals: which columns have nulls, which are wrong type

# --- Missing values ---
null_counts = df.isnull().sum()
null_pct    = (df.isnull().sum() / len(df) * 100).round(1)
null_report = pd.DataFrame({'null_count': null_counts, 'null_pct': null_pct})
print(null_report[null_report['null_count'] > 0])

# --- Duplicates ---
print(f"Duplicate order_ids: {df['order_id'].duplicated().sum()}")
print(f"Exact duplicate rows: {df.duplicated().sum()}")

# --- Logical validity checks ---
print(f"Negative amount_inr:    {(df['amount_inr'] < 0).sum()}")
print(f"Zero quantity orders:   {(df['quantity'] == 0).sum()}")
print(f"Future order dates:     {(df['order_date'] > pd.Timestamp.today()).sum()}")
print(f"Delivery before order:  {(df['delivery_date'] < df['order_date']).sum()}")

# --- Category consistency ---
print(df['status'].value_counts())
# Look for: 'Delivered' vs 'delivered', 'RETURNED' vs 'returned'

print(df['city'].value_counts().head(20))
# Look for: 'New Delhi' vs 'Delhi', 'Bengaluru' vs 'Bangalore'

# --- Numeric ranges ---
print(df[['amount_inr', 'quantity', 'delivery_days', 'rating']].describe())

# === NUMERIC COLUMNS ===

# Distribution of order values
plt.figure(figsize=(10, 4))
plt.hist(df['amount_inr'], bins=60, color='#1d4ed8', edgecolor='white', alpha=0.85)
plt.axvline(df['amount_inr'].median(), color='red', linestyle='--', linewidth=1.5,
            label=f"Median: ₹{df['amount_inr'].median():,.0f}")
plt.axvline(df['amount_inr'].mean(),   color='orange', linestyle='--', linewidth=1.5,
            label=f"Mean: ₹{df['amount_inr'].mean():,.0f}")
plt.title('Distribution of Order Values — India E-Commerce 2026', fontsize=13, fontweight='bold')
plt.xlabel('Order Amount (₹)');  plt.ylabel('Number of Orders')
plt.legend();  plt.tight_layout();  plt.show()

# Key stats
print(f"Mean:    ₹{df['amount_inr'].mean():,.0f}")
print(f"Median:  ₹{df['amount_inr'].median():,.0f}")
print(f"Std Dev: ₹{df['amount_inr'].std():,.0f}")
print(f"Skewness: {df['amount_inr'].skew():.2f}")   # >1 = right-skewed

# === CATEGORICAL COLUMNS ===

fig, axes = plt.subplots(2, 2, figsize=(14, 8))

for ax, col in zip(axes.flatten(), ['category', 'city', 'status', 'payment_method']):
    counts = df[col].value_counts().head(10)
    counts.plot(kind='barh', ax=ax, color='#7c3aed', edgecolor='white')
    ax.set_title(f'{col} — Top 10', fontweight='bold')
    ax.invert_yaxis()   # highest count at top

plt.tight_layout();  plt.show()

# === NUMERIC vs CATEGORICAL ===

# Order value distribution by category (box plots)
plt.figure(figsize=(12, 5))
order = df.groupby('category')['amount_inr'].median().sort_values(ascending=False).index
sns.boxplot(data=df, x='category', y='amount_inr', order=order, palette='Set2',
            flierprops={'marker': '.', 'markersize': 3})
plt.title('Order Value Distribution by Category', fontsize=13, fontweight='bold')
plt.xticks(rotation=30, ha='right');  plt.ylabel('Amount (₹)')
plt.tight_layout();  plt.show()

# Average order value AND return rate by category — grouped bar
cat_stats = df.groupby('category').agg(
    aov          = ('amount_inr', 'mean'),
    return_rate  = ('status',     lambda x: (x == 'returned').mean() * 100),
    order_count  = ('order_id',   'count'),
).round(1).sort_values('aov', ascending=False)
print(cat_stats)

# === NUMERIC vs NUMERIC ===

# Delivery time vs Return rate by city
city_stats = df.groupby('city').agg(
    avg_delivery_days = ('delivery_days', 'mean'),
    return_rate_pct   = ('status', lambda x: (x == 'returned').mean() * 100),
    revenue_cr        = ('amount_inr',   lambda x: x.sum() / 1e7),
).round(2)

plt.figure(figsize=(9, 6))
scatter = plt.scatter(
    city_stats['avg_delivery_days'],
    city_stats['return_rate_pct'],
    s=city_stats['revenue_cr'] * 40,   # size = revenue
    alpha=0.7, color='#dc2626', edgecolors='white', linewidth=0.5
)
for city, row in city_stats.iterrows():
    plt.annotate(city, (row['avg_delivery_days'], row['return_rate_pct']),
                 fontsize=8, xytext=(4, 4), textcoords='offset points')
plt.xlabel('Average Delivery Time (Days)');  plt.ylabel('Return Rate (%)')
plt.title('Delivery Time vs Return Rate by City\n(bubble size = revenue)', fontweight='bold')
plt.tight_layout();  plt.show()

# === CORRELATION MATRIX ===
numeric_df = df.select_dtypes(include=[np.number])
plt.figure(figsize=(7, 5))
sns.heatmap(numeric_df.corr(), annot=True, fmt='.2f', cmap='coolwarm', center=0,
            square=True, linewidths=0.5)
plt.title('Correlation Matrix — Numeric Variables', fontweight='bold')
plt.tight_layout();  plt.show()

# --- Monthly revenue trend ---
df['month'] = df['order_date'].dt.to_period('M')
monthly = df.groupby('month').agg(
    revenue     = ('amount_inr', 'sum'),
    orders      = ('order_id',   'count'),
    aov         = ('amount_inr', 'mean'),
).reset_index()
monthly['month_str'] = monthly['month'].astype(str)

fig, ax1 = plt.subplots(figsize=(12, 5))
ax2 = ax1.twinx()
ax1.bar(monthly['month_str'], monthly['revenue'] / 1e5, color='#1d4ed8', alpha=0.7, label='Revenue (₹ Lakh)')
ax2.plot(monthly['month_str'], monthly['aov'], color='#dc2626', marker='o', linewidth=2, label='AOV (₹)')
ax1.set_xlabel('Month');  ax1.set_ylabel('Revenue (₹ Lakh)');  ax2.set_ylabel('AOV (₹)')
ax1.set_title('Monthly Revenue and Average Order Value', fontweight='bold')
plt.xticks(rotation=45, ha='right')
fig.legend(loc='upper left', bbox_to_anchor=(0.1, 0.9));  plt.tight_layout();  plt.show()

# --- Day of week pattern ---
df['weekday'] = df['order_date'].dt.day_name()
day_order = ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
day_stats = df.groupby('weekday')['amount_inr'].sum().reindex(day_order)

plt.figure(figsize=(8, 4))
day_stats.plot(kind='bar', color=['#1d4ed8']*5 + ['#dc2626']*2, edgecolor='white')
plt.title('Revenue by Day of Week', fontweight='bold')
plt.ylabel('Revenue (₹)');  plt.xticks(rotation=30, ha='right')
plt.tight_layout();  plt.show()

# --- MoM growth ---
monthly['prev_revenue'] = monthly['revenue'].shift(1)
monthly['mom_pct'] = ((monthly['revenue'] - monthly['prev_revenue'])
                      / monthly['prev_revenue'] * 100).round(1)
print(monthly[['month_str', 'revenue', 'mom_pct']].dropna())

# === IQR method ===
Q1  = df['amount_inr'].quantile(0.25)
Q3  = df['amount_inr'].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR

outliers = df[(df['amount_inr'] < lower_bound) | (df['amount_inr'] > upper_bound)]
print(f"Outliers detected: {len(outliers):,} rows ({len(outliers)/len(df)*100:.1f}%)")

# Examine high-value outliers specifically
high_outliers = df[df['amount_inr'] > upper_bound]
print("\nHigh-value outlier breakdown:")
print(high_outliers.groupby('category').agg(
    count     = ('order_id',   'count'),
    avg_value = ('amount_inr', 'mean'),
    min_value = ('amount_inr', 'min'),
    max_value = ('amount_inr', 'max'),
).sort_values('avg_value', ascending=False))

# === Z-score method (for normally distributed data) ===
from scipy import stats
df['amount_zscore'] = np.abs(stats.zscore(df['amount_inr']))
zscore_outliers = df[df['amount_zscore'] > 3]
print(f"\nZ-score outliers (|z|>3): {len(zscore_outliers):,} rows")

# === Visualise outliers ===
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
ax1.hist(df['amount_inr'], bins=60, color='#1d4ed8', edgecolor='white')
ax1.axvline(upper_bound, color='red', linestyle='--', label=f'IQR upper: ₹{upper_bound:,.0f}')
ax1.set_title('Order Value Distribution with IQR Boundary', fontweight='bold')
ax1.legend()
df.boxplot(column='amount_inr', by='category', ax=ax2, figsize=(12,4))
ax2.set_title('Outliers by Category (box plot)', fontweight='bold')
plt.tight_layout();  plt.show()

What is EDA and why is it important?

EDA (Exploratory Data Analysis) is the process of examining a dataset before formal analysis or modelling — to understand its structure, spot problems, and discover patterns. It was popularised by statistician John Tukey in the 1970s and remains the most critical step in any data project. EDA is important because: (1) it reveals data quality issues (nulls, duplicates, wrong data types, inconsistent categories) that would corrupt your analysis if not caught; (2) it surfaces the shape of distributions — whether data is normally distributed, skewed, or bimodal — which affects which statistical methods you can apply; (3) it reveals relationships between variables before you build any model; (4) it often answers the business question directly, without needing a complex model. In real analyst work, EDA is not a one-time step — you return to it whenever your results look unexpected.

What are the main steps of EDA?

EDA follows six main steps: (1) Understand the dataset structure — shape, column names, data types, and a sample of rows; (2) Data quality audit — count nulls, duplicates, and check for values that are technically valid but logically wrong (negative prices, future dates on past orders); (3) Univariate analysis — examine each variable individually: distribution of numeric columns (mean, median, std, histogram), frequency counts of categorical columns; (4) Bivariate analysis — examine relationships between pairs of variables: numeric vs numeric (scatter plot, correlation), numeric vs categorical (box plots by group, grouped bar charts); (5) Time-based analysis — if date columns exist, examine trends, seasonality, and day-of-week patterns; (6) Outlier analysis — identify values that fall far from the rest and decide whether they are errors or legitimate edge cases. The order matters but is flexible — findings in one step often send you back to an earlier step.

What Python libraries are used for EDA?

The standard Python EDA stack: pandas for data loading, cleaning, and aggregation (groupby, value_counts, describe, isnull); matplotlib for basic charts (histogram, line chart, scatter); seaborn for statistical charts with better defaults (boxplot, heatmap, pairplot, violinplot); numpy for numerical operations. Some analysts also use: plotly for interactive charts (useful when you want to zoom in or hover over data points); ydata-profiling (formerly pandas-profiling) for automated EDA reports that generate summary statistics, distributions, and correlation matrices for every column with one function call. For beginners, start with pandas + seaborn — they cover 95% of EDA needs.

How is EDA different from data cleaning?

Data cleaning is the process of fixing data quality problems — filling nulls, removing duplicates, fixing data types, standardising text. EDA is the process of understanding the data — discovering its structure, distributions, relationships, and patterns. In practice, they overlap: you cannot do EDA without first knowing what is in the data (which requires some cleaning), and you cannot clean data intelligently without first understanding it (which requires some exploration). The practical workflow: run a quick initial EDA to discover what needs cleaning, clean the data, then run a deeper EDA on the cleaned data. Data cleaning is covered in Chapter 12 of this series.