Advanced Python Data Analysis

Method chaining, pipe(), custom aggregations, memory optimisation, multi-index, cross-tabs, Plotly interactive charts, and scipy statistical testing.

import pandas as pd

# Unchained (verbose, creates three throwaway variables)
df1 = df.dropna(subset=['revenue'])
df2 = df1[df1['revenue'] > 0]
df3 = df2.assign(margin=lambda x: (x['revenue'] - x['cost']) / x['revenue'])
result = df3.sort_values('margin', ascending=False)

# Chained (one expression, same result)
result = (
    df
    .dropna(subset=['revenue'])
    .query('revenue > 0')
    .assign(margin=lambda x: (x['revenue'] - x['cost']) / x['revenue'])
    .sort_values('margin', ascending=False)
)

# assign() adds or overwrites columns and returns a new DataFrame
# Lambda receives the DataFrame at that point in the chain

result = (
    df
    .assign(
        revenue_k    = lambda x: x['revenue'] / 1000,
        margin_pct   = lambda x: (x['revenue'] - x['cost']) / x['revenue'] * 100,
        is_profitable= lambda x: x['margin_pct'] > 0,
        month        = lambda x: x['order_date'].dt.to_period('M'),
    )
)

# Chain multiple assign() calls when later columns depend on earlier ones
result = (
    df
    .assign(revenue_k = lambda x: x['revenue'] / 1000)
    .assign(revenue_tier = lambda x: pd.cut(
        x['revenue_k'],
        bins=[0, 10, 50, 200, float('inf')],
        labels=['Small', 'Medium', 'Large', 'Enterprise']
    ))
)

# pipe() calls a function with the DataFrame as the first argument
# This lets you include any custom function in a chain

def remove_test_accounts(df, email_col='email'):
    """Drop rows where email contains 'test' or '@example'."""
    mask = (
        df[email_col].str.contains('test', case=False, na=False) |
        df[email_col].str.contains('@example', case=False, na=False)
    )
    return df[~mask]

def add_customer_tier(df, revenue_col='ltv'):
    bins   = [0, 1000, 5000, 20000, float('inf')]
    labels = ['Bronze', 'Silver', 'Gold', 'Platinum']
    return df.assign(tier=pd.cut(df[revenue_col], bins=bins, labels=labels))

def log_shape(df, label=''):
    """Debug step — prints shape without breaking the chain."""
    print(f"{label}: {df.shape}")
    return df

result = (
    raw_df
    .pipe(remove_test_accounts, email_col='user_email')
    .pipe(log_shape, label='after removing test accounts')
    .dropna(subset=['ltv'])
    .pipe(add_customer_tier, revenue_col='ltv')
    .query('tier != "Bronze"')
    .reset_index(drop=True)
)

# Named aggregations: control output column names exactly
summary = (
    df
    .groupby('region')
    .agg(
        total_revenue  = ('revenue', 'sum'),
        avg_order_value= ('revenue', 'mean'),
        order_count    = ('order_id', 'count'),
        unique_customers=('customer_id', 'nunique'),
        max_single_order=('revenue', 'max'),
        first_order_date=('order_date', 'min'),
        last_order_date =('order_date', 'max'),
    )
    .round(2)
    .reset_index()
)

# Pass any callable — receives the group Series
def pct_above_threshold(s, threshold=1000):
    return (s > threshold).mean() * 100

def iqr(s):
    return s.quantile(0.75) - s.quantile(0.25)

summary = (
    df
    .groupby('category')
    .agg(
        revenue_iqr      = ('revenue', iqr),
        pct_large_orders = ('revenue', lambda s: pct_above_threshold(s, 1000)),
    )
)

# transform() returns a Series with the same index as the input
# Use it to add group-level stats as new columns without collapsing rows

df = df.assign(
    region_total   = df.groupby('region')['revenue'].transform('sum'),
    region_avg     = df.groupby('region')['revenue'].transform('mean'),
    # % contribution of this row to its region total
    pct_of_region  = lambda x: x['revenue'] / x['region_total'] * 100,
    # Rank within region (highest revenue = rank 1)
    rank_in_region = df.groupby('region')['revenue'].transform(
        lambda s: s.rank(ascending=False, method='dense')
    ),
)

# Keep only groups (regions) with more than 100 orders
large_regions = df.groupby('region').filter(lambda g: len(g) > 100)

# Keep only groups where the average revenue exceeds 5000
high_value_regions = df.groupby('region').filter(
    lambda g: g['revenue'].mean() > 5000
)

# Group by multiple dimensions — creates a MultiIndex result
pivot = (
    df
    .groupby(['region', 'category'])['revenue']
    .agg(['sum', 'count', 'mean'])
    .round(0)
)

# Access a specific region
pivot.loc['North']

# Flatten MultiIndex columns after agg
pivot.columns = ['_'.join(col).strip() for col in pivot.columns]

# Reset to flat DataFrame
pivot = pivot.reset_index()

import pandas as pd

# Basic count crosstab
ct = pd.crosstab(df['region'], df['category'])

# With margins (row/column totals)
ct_with_totals = pd.crosstab(
    df['region'], df['category'],
    margins=True, margins_name='Total'
)

# Normalise to proportions
ct_pct = pd.crosstab(
    df['region'], df['category'],
    normalize='index'   # 'index'=row%, 'columns'=col%, 'all'=grand total%
).round(3)

# With an aggregation value (not just count)
ct_revenue = pd.crosstab(
    df['region'], df['category'],
    values=df['revenue'], aggfunc='sum'
)

import pandas as pd

def optimise_dataframe(df):
    """Downcast numeric types and categorise low-cardinality strings."""
    df = df.copy()

    for col in df.select_dtypes(include=['int64']).columns:
        df[col] = pd.to_numeric(df[col], downcast='integer')

    for col in df.select_dtypes(include=['float64']).columns:
        df[col] = pd.to_numeric(df[col], downcast='float')

    for col in df.select_dtypes(include=['object']).columns:
        if df[col].nunique() / len(df) < 0.05:  # < 5% unique = low cardinality
            df[col] = df[col].astype('category')

    return df

# Check memory before and after
print(f"Before: {df.memory_usage(deep=True).sum() / 1e6:.1f} MB")
df = optimise_dataframe(df)
print(f"After:  {df.memory_usage(deep=True).sum() / 1e6:.1f} MB")

# Read only what you need from the start
df = pd.read_csv(
    'large_file.csv',
    usecols=['order_id', 'revenue', 'region', 'order_date'],
    dtype={'region': 'category', 'order_id': 'int32'},
    parse_dates=['order_date'],
)

import pandas as pd
import numpy as np

# SLOW: apply() with a Python function (row-by-row)
df['tier'] = df['revenue'].apply(
    lambda x: 'High' if x > 10000 else ('Mid' if x > 1000 else 'Low')
)

# FAST: np.select() — vectorised conditional
conditions  = [df['revenue'] > 10000, df['revenue'] > 1000]
choices     = ['High', 'Mid']
df['tier']  = np.select(conditions, choices, default='Low')

# FAST: pd.cut() for binning
df['tier'] = pd.cut(
    df['revenue'],
    bins=[0, 1000, 10000, float('inf')],
    labels=['Low', 'Mid', 'High']
)

# FAST: str accessor for string operations (vectorised)
df['region_clean'] = df['region'].str.upper().str.strip()
df['has_discount'] = df['promo_code'].str.startswith('DISC')

# Rule of thumb: if you can express it with Series/array operations
# or numpy — do that. Reserve apply() for logic that genuinely
# cannot be vectorised (API calls, custom objects, complex branching).

import plotly.express as px
import pandas as pd

monthly = (
    df
    .assign(month=lambda x: x['order_date'].dt.to_period('M').dt.to_timestamp())
    .groupby(['month', 'region'])['revenue']
    .sum()
    .reset_index()
)

fig = px.line(
    monthly,
    x='month', y='revenue', color='region',
    title='Monthly Revenue by Region',
    labels={'revenue': 'Revenue (₹)', 'month': ''},
    template='plotly_white',
)
fig.update_layout(legend_title_text='Region', hovermode='x unified')
fig.show()

fig = px.scatter(
    df,
    x='spend', y='revenue',
    size='order_count',       # bubble size
    color='category',         # colour by dimension
    hover_data=['region', 'margin_pct'],
    title='Spend vs Revenue by Category',
    trendline='ols',          # add regression line per group
    template='plotly_white',
    labels={'spend': 'Marketing Spend (₹)', 'revenue': 'Revenue (₹)'},
)
fig.show()

from plotly.subplots import make_subplots
import plotly.graph_objects as go

fig = make_subplots(
    rows=2, cols=2,
    subplot_titles=['Revenue by Region', 'Order Count Trend',
                    'Margin Distribution', 'Top 10 Products'],
)

# Row 1 Col 1 — bar chart
fig.add_trace(
    go.Bar(x=region_df['region'], y=region_df['revenue'], name='Revenue'),
    row=1, col=1
)

# Row 1 Col 2 — line chart
fig.add_trace(
    go.Scatter(x=trend_df['date'], y=trend_df['orders'], name='Orders'),
    row=1, col=2
)

# Row 2 Col 1 — histogram
fig.add_trace(
    go.Histogram(x=df['margin_pct'], name='Margin %', nbinsx=30),
    row=2, col=1
)

fig.update_layout(height=700, title_text='Executive Dashboard', showlegend=False)
fig.show()

from scipy import stats
import pandas as pd

group_a = df[df['variant'] == 'A']['conversion_rate']
group_b = df[df['variant'] == 'B']['conversion_rate']

# Independent two-sample t-test
t_stat, p_value = stats.ttest_ind(group_a, group_b, equal_var=False)

print(f"t-statistic: {t_stat:.4f}")
print(f"p-value:     {p_value:.4f}")
print(f"Result: {'Significant difference' if p_value < 0.05 else 'No significant difference'} at α=0.05")

# Effect size (Cohen's d)
pooled_std = ((group_a.std()**2 + group_b.std()**2) / 2) ** 0.5
cohens_d   = (group_a.mean() - group_b.mean()) / pooled_std
print(f"Cohen's d: {cohens_d:.3f}  ({'large' if abs(cohens_d)>0.8 else 'medium' if abs(cohens_d)>0.5 else 'small'} effect)")

from scipy.stats import chi2_contingency
import pandas as pd

# Contingency table: payment method vs order outcome
ct = pd.crosstab(df['payment_method'], df['order_status'])

chi2, p, dof, expected = chi2_contingency(ct)
print(f"Chi-square: {chi2:.4f}")
print(f"p-value:    {p:.4f}")
print(f"Degrees of freedom: {dof}")
print(f"Association: {'Significant' if p < 0.05 else 'Not significant'} at α=0.05")

import pandas as pd
from scipy.stats import pearsonr, spearmanr

# Pearson (linear relationships, assumes normal distribution)
r, p = pearsonr(df['ad_spend'], df['revenue'])
print(f"Pearson r={r:.3f}, p={p:.4f}")

# Spearman (monotonic relationships, no normality assumption — more robust)
rho, p = spearmanr(df['ad_spend'], df['revenue'])
print(f"Spearman ρ={rho:.3f}, p={p:.4f}")

# Full correlation matrix (Pandas built-in)
corr_matrix = df[['revenue', 'ad_spend', 'sessions', 'conversion_rate']].corr()

# Heatmap with Plotly
import plotly.express as px
fig = px.imshow(
    corr_matrix, text_auto='.2f', aspect='auto',
    color_continuous_scale='RdBu_r', zmin=-1, zmax=1,
    title='Correlation Matrix'
)
fig.show()

import pandas as pd
import time

# Time a single operation
start = time.perf_counter()
result = df.groupby('region')['revenue'].sum()
elapsed = time.perf_counter() - start
print(f"groupby sum: {elapsed*1000:.2f}ms")

# In Jupyter — use %%timeit magic
# %%timeit
# df.groupby('region')['revenue'].sum()

# Profile memory of a specific expression
import tracemalloc
tracemalloc.start()
result = df.merge(lookup, on='product_id', how='left')
current, peak = tracemalloc.get_traced_memory()
tracemalloc.stop()
print(f"Peak memory: {peak / 1e6:.1f} MB")

# Common bottlenecks and fixes:
# SLOW: df.apply(fn, axis=1)      → use vectorised ops or np.select
# SLOW: iterrows() / itertuples() → use vectorised ops
# SLOW: repeated concatenation    → collect list of dfs, then pd.concat()
# SLOW: str ops on object dtype   → use .str accessor (vectorised)
# SLOW: full scan on large df     → filter early, set_index for repeated key lookups