Correlation & Regression Analysis for Data Analysts

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

df = pd.read_csv('india_ecommerce_orders.csv')

# === PEARSON CORRELATION ===
# Best for: normally distributed, continuous variables, linear relationship

# Single pair
r, p_value = stats.pearsonr(df['delivery_days'].dropna(), df['return_rate'].dropna())
print(f"Delivery days vs Return rate: r = {r:.3f}, p = {p_value:.4f}")

# Full correlation matrix
numeric_cols = ['amount_inr', 'quantity', 'delivery_days', 'rating', 'discount_pct']
corr_matrix  = df[numeric_cols].corr(method='pearson')
print(corr_matrix.round(2))

# Heatmap of correlation matrix
plt.figure(figsize=(8, 6))
mask = np.triu(np.ones_like(corr_matrix, dtype=bool))  # hide upper triangle
sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm',
            center=0, mask=mask, square=True, linewidths=0.5,
            cbar_kws={'shrink': 0.8})
plt.title('Correlation Matrix — Indian E-Commerce Orders', fontweight='bold')
plt.tight_layout(); plt.show()

# === SPEARMAN CORRELATION ===
# Best for: skewed data, ordinal variables, monotonic (not necessarily linear) relationships
# Use for: amount_inr (right-skewed), rating (ordinal 1-5)

rho, p_val = stats.spearmanr(df['amount_inr'], df['quantity'])
print(f"Order value vs Quantity (Spearman): rho = {rho:.3f}, p = {p_val:.4f}")

# === SCATTER PLOT WITH REGRESSION LINE ===
city_stats = df.groupby('city').agg(
    avg_delivery_days = ('delivery_days', 'mean'),
    return_rate_pct   = ('status', lambda x: (x == 'returned').mean() * 100),
    order_count       = ('order_id', 'count'),
).reset_index()

plt.figure(figsize=(9, 6))
plt.scatter(city_stats['avg_delivery_days'], city_stats['return_rate_pct'],
            s=city_stats['order_count'] / 50, alpha=0.7, color='#0f766e', edgecolors='white')

# Add regression line
m, b = np.polyfit(city_stats['avg_delivery_days'], city_stats['return_rate_pct'], 1)
x_line = np.linspace(city_stats['avg_delivery_days'].min(),
                     city_stats['avg_delivery_days'].max(), 100)
plt.plot(x_line, m * x_line + b, 'r--', linewidth=2,
         label=f'Trend line: y = {m:.1f}x + {b:.1f}')

for _, row in city_stats.iterrows():
    plt.annotate(row['city'], (row['avg_delivery_days'], row['return_rate_pct']),
                 fontsize=8, xytext=(4, 3), textcoords='offset points')

r_val, _ = stats.pearsonr(city_stats['avg_delivery_days'], city_stats['return_rate_pct'])
plt.xlabel('Average Delivery Time (Days)'); plt.ylabel('Return Rate (%)')
plt.title(f'Delivery Time vs Return Rate by City  (r = {r_val:.2f})', fontweight='bold')
plt.legend(); plt.tight_layout(); plt.show()

from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score, mean_absolute_error
import numpy as np

# Business question: How does monthly ad spend predict monthly revenue?
monthly = df.groupby(df['order_date'].dt.to_period('M')).agg(
    revenue_lakhs = ('amount_inr', lambda x: x.sum() / 1e5),
    ad_spend_lakhs = ('ad_spend', lambda x: x.sum() / 1e5),
).reset_index()

X = monthly[['ad_spend_lakhs']]   # must be 2D for sklearn
y = monthly['revenue_lakhs']

# Fit the model
model = LinearRegression()
model.fit(X, y)

print(f"Intercept (a):  {model.intercept_:.2f} lakh")
print(f"Coefficient (b): {model.coef_[0]:.2f}")
print(f"R-squared:       {model.score(X, y):.3f}")

# Interpretation:
# Intercept = 42.3: even with ₹0 ad spend, base revenue is ₹42.3 lakh (organic/repeat customers)
# Coefficient = 3.8: each ₹1 lakh of ad spend is associated with ₹3.8 lakh additional revenue
# R-squared = 0.67: ad spend explains 67% of monthly revenue variation

# Predict revenue for a ₹10 lakh ad spend
predicted = model.predict([[10]])
print(f"\nPredicted revenue for ₹10L ad spend: ₹{predicted[0]:.1f} lakh")

# Residual analysis (check model fit)
y_pred     = model.predict(X)
residuals  = y - y_pred
mae        = mean_absolute_error(y, y_pred)

print(f"\nMean Absolute Error: ₹{mae:.1f} lakh")
print(f"Residuals range: {residuals.min():.1f} to {residuals.max():.1f} lakh")

# Plot actual vs predicted
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.scatter(X, y, color='#0f766e', alpha=0.7, label='Actual')
plt.plot(X, y_pred, color='red', linewidth=2, label='Regression line')
plt.xlabel('Ad Spend (₹ Lakh)'); plt.ylabel('Revenue (₹ Lakh)')
plt.title('Ad Spend vs Revenue', fontweight='bold'); plt.legend()

plt.subplot(1, 2, 2)
plt.scatter(y_pred, residuals, color='#7c3aed', alpha=0.7)
plt.axhline(0, color='red', linestyle='--')
plt.xlabel('Predicted Revenue'); plt.ylabel('Residuals')
plt.title('Residual Plot', fontweight='bold')
plt.tight_layout(); plt.show()

import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import r2_score
import statsmodels.api as sm   # for detailed stats output

# Predict order value from: category, city tier, discount %, and is_weekend
df['is_weekend'] = df['order_date'].dt.dayofweek >= 5

# Encode categoricals
le_cat  = LabelEncoder()
le_tier = LabelEncoder()
df['category_enc'] = le_cat.fit_transform(df['category'])
df['city_tier_enc'] = le_tier.fit_transform(df['city_tier'])  # Tier 1/2/3

features = ['category_enc', 'city_tier_enc', 'discount_pct', 'is_weekend']
X = df[features].fillna(0)
y = df['amount_inr']

# sklearn: quick R-squared
model = LinearRegression().fit(X, y)
print(f"R-squared: {model.score(X, y):.3f}")
for feat, coef in zip(features, model.coef_):
    print(f"  {feat:20s}: {coef:+.1f}")

# statsmodels: detailed output with p-values and confidence intervals
X_sm = sm.add_constant(X)
ols  = sm.OLS(y, X_sm).fit()
print(ols.summary())

# Key interpretation from output:
# discount_pct coef = -8.4 → each 1% discount is associated with ₹8.4 lower order value
#   (customers who buy on discount tend to buy cheaper items)
# is_weekend coef = +124 → weekend orders are ₹124 higher on average
# Adjusted R-squared = 0.31 → model explains 31% of order value variation
# Low R-squared is expected — many factors (specific product, promotions) not captured

# Adjusted R-squared (always use this when comparing models with different numbers of features)
n = len(y); k = X.shape[1]
adj_r2 = 1 - (1 - model.score(X, y)) * (n - 1) / (n - k - 1)
print(f"Adjusted R-squared: {adj_r2:.3f}")

What is the difference between correlation and causation?

Correlation measures the strength and direction of a linear relationship between two variables — whether they tend to move together. Causation means one variable directly causes the other to change. Correlation never proves causation. A classic example: ice cream sales and drowning rates are positively correlated — both increase in summer. But ice cream does not cause drowning; a third variable (hot weather) drives both. In business analysis, you will frequently find correlations: cities with higher average income tend to have higher AOV, longer delivery times correlate with higher return rates. These are useful observations for targeting and operations — but they do not prove that improving delivery time will reduce returns. To establish causation, you need a controlled experiment (A/B test) or a carefully designed quasi-experimental study. Always use language like "associated with" or "correlates with" rather than "causes" unless you have experimental evidence.

What is R-squared in regression and what is a good value?

R-squared (coefficient of determination) measures what proportion of the variance in the dependent variable is explained by the independent variables in the regression model. R-squared ranges from 0 to 1 (or 0% to 100%). An R-squared of 0.72 means the model explains 72% of the variation in the outcome; the remaining 28% is unexplained. What counts as "good" depends entirely on the domain: in physical sciences where conditions are controlled, R-squared above 0.95 is common. In business and social science, R-squared of 0.4–0.7 is often acceptable because human behaviour involves many unobservable factors. For an Indian e-commerce model predicting order value from city and category, R-squared of 0.35 might be quite reasonable. Never optimise solely for high R-squared — adding irrelevant variables always increases R-squared (use Adjusted R-squared instead, which penalises unnecessary variables).

What is Pearson correlation and when should you use Spearman instead?

Pearson correlation (r) measures the strength of a linear relationship between two continuous variables. It ranges from -1 (perfect negative linear relationship) to +1 (perfect positive linear relationship). 0 means no linear relationship. Pearson assumes both variables are normally distributed and the relationship is linear. Use Spearman correlation when: (1) the data is ordinal (customer satisfaction ratings 1–5, NPS scores); (2) the data is not normally distributed (income, order values — typically right-skewed); (3) the relationship is monotonic but not linear (as X increases, Y consistently increases, but not at a constant rate). In Indian business analysis, order values and delivery times are typically skewed — Spearman is more appropriate than Pearson for these. Use Pearson for normally distributed metrics like standardised test scores, manufacturing tolerances, or log-transformed financial data.

Do data analysts in India need to know regression analysis?

Knowing the concepts of correlation and simple linear regression is expected of data analysts in India at the intermediate to senior level. The specific depth depends on the role: at analytics-heavy companies (fintech, e-commerce analytics teams, consulting firms), analysts are expected to run regression models, interpret coefficients and R-squared, and communicate findings to non-technical stakeholders. At traditional companies (FMCG, manufacturing, HR functions), the expectation is more conceptual — understanding what correlation means, not building full models. For interviews, you should be able to explain: what correlation coefficient means, the difference between correlation and causation, what R-squared measures, and what a regression coefficient tells you. Running a model in Python is a plus. The deeper machine learning applications (regularisation, feature selection) are more in the data scientist domain.