Python Data Science

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Load data
 df = pd.read_csv('data.csv', parse_dates=['date'])

# Basic exploration
print(df.info())
print(df.describe())
print(df.head())

# Missing values analysis
missing_pct = df.isna().mean() * 100
print(f"Missing values:\n{missing_pct[missing_pct > 0]}")

# Data type optimization
df['category'] = df['category'].astype('category')
df['date'] = pd.to_datetime(df['date'])

# Statistical summary by group
df.groupby('category').agg({
    'value': ['mean', 'std', 'min', 'max'],
    'count': 'sum'
})

# Correlation analysis
numeric_cols = df.select_dtypes(include=[np.number]).columns
correlation_matrix = df[numeric_cols].corr()
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm')
plt.title('Correlation Matrix')
plt.show()

from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix
import pandas as pd

# Load data
df = pd.read_csv('data.csv')
X = df.drop('target', axis=1)
y = df['target']

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# Define preprocessing
numeric_features = X.select_dtypes(include=['int64', 'float64']).columns
categorical_features = X.select_dtypes(include=['object', 'category']).columns

preprocessor = ColumnTransformer(
    transformers=[
        ('num', StandardScaler(), numeric_features),
        ('cat', OneHotEncoder(drop='first', sparse_output=False), categorical_features)
    ]
)

# Create pipeline
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42))
])

# Hyperparameter tuning
param_grid = {
    'classifier__n_estimators': [100, 200, 300],
    'classifier__max_depth': [10, 20, None]
}

grid_search = GridSearchCV(
    pipeline, param_grid, cv=5, scoring='f1', n_jobs=-1
)
grid_search.fit(X_train, y_train)

# Evaluate
best_model = grid_search.best_estimator_
y_pred = best_model.predict(X_test)
print(classification_report(y_test, y_pred))
print(f"Best CV score: {grid_search.best_score_:.4f}")

import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import pandas as pd

# Load data
df = pd.read_csv('sales_data.csv', parse_dates=['date'])

# Interactive scatter plot
fig = px.scatter(
    df, x='revenue', y='profit',
    color='category', size='quantity',
    hover_data=['product', 'date'],
    title='Revenue vs Profit by Category',
    labels={'revenue': 'Revenue ($)', 'profit': 'Profit ($)'}
)
fig.update_layout(template='plotly_dark')
fig.show()

# Time series with multiple traces
fig = make_subplots(
    rows=2, cols=1,
    subplot_titles=('Revenue Over Time', 'Profit Margin Over Time'),
    vertical_spacing=0.1
)

fig.add_trace(
    go.Scatter(x=df['date'], y=df['revenue'], name='Revenue', mode='lines+markers'),
    row=1, col=1
)

fig.add_trace(
    go.Scatter(x=df['date'], y=df['profit_margin'], name='Profit Margin', mode='lines'),
    row=2, col=1
)

fig.update_layout(height=600, title_text='Sales Analytics Dashboard')
fig.show()

from scipy import stats
from scipy.stats import ttest_ind, chi2_contingency, pearsonr
import statsmodels.api as sm
from statsmodels.tsa.seasonal import seasonal_decompose
import pandas as pd
import numpy as np

# Load data
df = pd.read_csv('experiment_data.csv')

# T-test for comparing two groups
control = df[df['group'] == 'control']['metric']
treatment = df[df['group'] == 'treatment']['metric']

t_stat, p_value = ttest_ind(control, treatment)
print(f"T-statistic: {t_stat:.4f}, P-value: {p_value:.4f}")

# Chi-square test for categorical data
contingency_table = pd.crosstab(df['category'], df['outcome'])
chi2, p_value, dof, expected = chi2_contingency(contingency_table)
print(f"Chi-square: {chi2:.4f}, P-value: {p_value:.4f}")

# Correlation analysis
corr, p_value = pearsonr(df['x'], df['y'])
print(f"Correlation: {corr:.4f}, P-value: {p_value:.4f}")

# Linear regression
X = sm.add_constant(df[['feature1', 'feature2']])
y = df['target']
model = sm.OLS(y, X).fit()
print(model.summary())

# Time series decomposition
df_ts = pd.read_csv('time_series.csv', parse_dates=['date'], index_col='date')
decomposition = seasonal_decompose(df_ts['value'], model='additive', period=12)
decomposition.plot()
plt.show()

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler, RobustScaler, LabelEncoder
from sklearn.impute import SimpleImputer, KNNImputer

# Load data
df = pd.read_csv('raw_data.csv')

# Handle missing values
# For numeric columns
numeric_imputer = SimpleImputer(strategy='median')
numeric_cols = df.select_dtypes(include=[np.number]).columns
df[numeric_cols] = numeric_imputer.fit_transform(df[numeric_cols])

# For categorical columns
categorical_imputer = SimpleImputer(strategy='most_frequent')
categorical_cols = df.select_dtypes(include=['object']).columns
df[categorical_cols] = categorical_imputer.fit_transform(df[categorical_cols])

# Feature engineering
df['date'] = pd.to_datetime(df['date'])
df['year'] = df['date'].dt.year
df['month'] = df['date'].dt.month
df['day_of_week'] = df['date'].dt.dayofweek
df['is_weekend'] = df['day_of_week'].isin([5, 6]).astype(int)

# Create interaction features
df['feature_interaction'] = df['feature1'] * df['feature2']
df['feature_ratio'] = df['feature1'] / (df['feature2'] + 1e-6)

# Binning numeric features
df['age_group'] = pd.cut(df['age'], bins=[0, 25, 50, 75, 100], labels=['Young', 'Adult', 'Senior', 'Elderly'])

# Encoding categorical variables
label_encoder = LabelEncoder()
df['category_encoded'] = label_encoder.fit_transform(df['category'])

# Scaling features
scaler = RobustScaler()
df[['feature1', 'feature2']] = scaler.fit_transform(df[['feature1', 'feature2']])

from sklearn.model_selection import cross_val_score, learning_curve, validation_curve
from sklearn.metrics import (accuracy_score, precision_score, recall_score, f1_score,
                             roc_auc_score, confusion_matrix, classification_report,
                             mean_squared_error, r2_score)
import matplotlib.pyplot as plt
import numpy as np

# Classification metrics
y_true = [0, 1, 1, 0, 1, 0, 1, 1, 0, 1]
y_pred = [0, 1, 1, 0, 0, 0, 1, 1, 0, 1]

y_pred_proba = [0.1, 0.9, 0.8, 0.2, 0.4, 0.3, 0.95, 0.85, 0.15, 0.9]

print(f"Accuracy: {accuracy_score(y_true, y_pred):.4f}")
print(f"Precision: {precision_score(y_true, y_pred):.4f}")
print(f"Recall: {recall_score(y_true, y_pred):.4f}")
print(f"F1 Score: {f1_score(y_true, y_pred):.4f}")
print(f"ROC AUC: {roc_auc_score(y_true, y_pred_proba):.4f}")
print(f"\nConfusion Matrix:\n{confusion_matrix(y_true, y_pred)}")
print(f"\nClassification Report:\n{classification_report(y_true, y_pred)}")

# Cross-validation
cv_scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
print(f"CV Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std() * 2:.4f})")

# Learning curve
train_sizes, train_scores, val_scores = learning_curve(
    model, X, y, cv=5, n_jobs=-1,
    train_sizes=np.linspace(0.1, 1.0, 10)
)

plt.figure(figsize=(10, 6))
plt.plot(train_sizes, train_scores.mean(axis=1), 'o-', label='Training Score')
plt.plot(train_sizes, val_scores.mean(axis=1), 'o-', label='Validation Score')
plt.xlabel('Training Set Size')
plt.ylabel('Score')
plt.title('Learning Curve')
plt.legend()
plt.grid(True)
plt.show()

import pandas as pd
import numpy as np
from datetime import datetime
import json
import pickle

# Set random seed for reproducibility
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)

# Log experiment parameters
experiment_config = {
    'timestamp': datetime.now().isoformat(),
    'random_seed': RANDOM_SEED,
    'model_type': 'RandomForest',
    'hyperparameters': {
        'n_estimators': 100,
        'max_depth': 20,
        'min_samples_split': 2
    },
    'data_version': 'v1.2.3',
    'features': ['feature1', 'feature2', 'feature3']
}

# Save configuration
with open('experiment_config.json', 'w') as f:
    json.dump(experiment_config, f, indent=2)

# Load and process data
df = pd.read_csv('data.csv')

# Data versioning
print(f"Data shape: {df.shape}")
print(f"Data hash: {pd.util.hash_pandas_object(df).sum()}")

# Train model
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(
    n_estimators=experiment_config['hyperparameters']['n_estimators'],
    max_depth=experiment_config['hyperparameters']['max_depth'],
    random_state=RANDOM_SEED
)
model.fit(X_train, y_train)

# Save model with metadata
model_metadata = {
    'model_type': 'RandomForest',
    'trained_at': datetime.now().isoformat(),
    'training_samples': len(X_train),
    'feature_names': list(X_train.columns),
    'experiment_config': experiment_config
}

with open('model.pkl', 'wb') as f:
    pickle.dump(model, f)

with open('model_metadata.json', 'w') as f:
    json.dump(model_metadata, f, indent=2)

print(f"Model saved with metadata: {model_metadata}")