💰 GCP Cost Optimization - Advanced

Introduction

Advanced cost optimization strategies enable tracking costs at granular levels, predicting future costs, detecting anomalies, and implementing automated cost controls.

Key Learning Outcomes

By the end of this lesson, you'll understand:

• Cost anomaly detection

• Chargeback and cost allocation

• Automated cost controls

• RI purchasing optimization

• Multi-tenancy cost management

• Cost forecasting with ML

• Resource optimization recommendations

Cost Anomaly Detection

from google.cloud import bigquery
from scipy import stats
import pandas as pd

class CostAnomalyDetector:
  def __init__(self, project_id):
    self.client = bigquery.Client(project=project_id)
  
  def detect_anomalies(self, days=30, z_score_threshold=2.5):
    query = f"""
    SELECT
      DATE(usage_start_time) as date,
      SUM(cost) as daily_cost
    FROM `billing_export`
    WHERE usage_start_time >= DATE_SUB(CURRENT_DATE(), INTERVAL {days} DAY)
    GROUP BY date
    ORDER BY date
    """
    
    df = self.client.query(query).to_pandas()
    
    # Calculate z-score
    df['z_score'] = stats.zscore(df['daily_cost'])
    df['is_anomaly'] = abs(df['z_score']) > z_score_threshold
    
    # Alert on anomalies
    anomalies = df[df['is_anomaly']]
    if not anomalies.empty:
      self.alert_anomalies(anomalies)
    
    return anomalies
  
  def alert_anomalies(self, anomalies):
    for _, row in anomalies.iterrows():
      print(f"""
      ALERT: Cost anomaly detected
      Date: {row['date']}
      Cost: ${row['daily_cost']:.2f}
      Z-Score: {row['z_score']:.2f}
      """)

Chargeback Model Implementation

class ChargebackCalculator:
  def calculate_department_costs(self, start_date, end_date):
    query = f"""
    SELECT
      labels.value as department,
      service.description,
      SUM(cost) as total_cost,
      SUM(usage.amount) as total_usage
    FROM `billing_export`
    WHERE
      _TABLE_SUFFIX BETWEEN
      FORMAT_DATE('%Y%m%d', '{start_date}')
      AND FORMAT_DATE('%Y%m%d', '{end_date}')
      AND labels.key = 'department'
    GROUP BY department, service.description
    """
    
    df = self.client.query(query).to_pandas()
    return df
  
  def allocate_shared_costs(self, shared_cost, allocation_key):
    """Allocate shared costs (e.g., load balancer) based on usage"""
    query = f"""
    SELECT
      {allocation_key},
      SUM(usage.amount) as usage
    FROM `billing_export`
    WHERE service.description IN ('Cloud Load Balancing', 'Cloud CDN')
    GROUP BY {allocation_key}
    """
    
    df = self.client.query(query).to_pandas()
    df['allocated_cost'] = (df['usage'] / df['usage'].sum()) * shared_cost
    
    return df

Automated Cost Controls

# Set up budget alerts in GCP Console
gcloud billing budgets create \
  --billing-account=ACCOUNT_ID \
  --display-name="Department Budget" \
  --budget-amount=10000 \
  --threshold-rule=percent=50 \
  --threshold-rule=percent=100 \
  --labels=department=engineering

Automated spending limits with Cloud Functions:

exports.enforceBudgetLimit = functions.pubsub
  .topic('billing-alerts')
  .onPublish(async (message) => {
    const alert = JSON.parse(Buffer.from(message.data, 'base64').toString());
    
    if (alert.budgetAmountExceeded) {
      // Stop resource-intensive services
      await disableAutoscaling('high-cost-service');
      await deleteUnusedResources();
      
      // Notify team
      await sendAlert(`Budget exceeded: ${alert.budgetDisplayName}`);
    }
  });

Reserved Instance Purchasing

class RIPurchaseOptimizer:
  def analyze_commitment_opportunity(self):
    query = """
    SELECT
      service.description,
      resource.region,
      SUM(cost) as monthly_cost,
      COUNT(DISTINCT DATE(usage_start_time)) as days_used
    FROM `billing_export`
    WHERE
      _TABLE_SUFFIX = FORMAT_DATE('%Y%m%d', CURRENT_DATE() - 1)
      AND service.description IN ('Compute Engine', 'Cloud SQL')
      AND resource.global_resources = False
    GROUP BY service.description, resource.region
    HAVING days_used > 25  # 25+ days used
    """
    
    df = self.client.query(query).to_pandas()
    
    # Calculate RI savings
    df['1yr_ri_cost'] = df['monthly_cost'] * 12 * 0.75  # 25% discount
    df['3yr_ri_cost'] = df['monthly_cost'] * 36 * 0.48  # 52% discount
    df['1yr_savings'] = (df['monthly_cost'] * 12) - df['1yr_ri_cost']
    df['3yr_savings'] = (df['monthly_cost'] * 36) - df['3yr_ri_cost']
    
    return df.sort_values('1yr_savings', ascending=False)

Cost Forecasting with ML

from sklearn.linear_model import LinearRegression
import numpy as np

class CostForecaster:
  def forecast_costs(self, forecast_days=30):
    # Get historical costs
    query = """
    SELECT
      DATE(usage_start_time) as date,
      SUM(cost) as daily_cost
    FROM `billing_export`
    WHERE usage_start_time >= DATE_SUB(CURRENT_DATE(), INTERVAL 90 DAY)
    GROUP BY date
    """
    
    df = self.client.query(query).to_pandas()
    
    # Prepare data
    X = np.arange(len(df)).reshape(-1, 1)
    y = df['daily_cost'].values
    
    # Train model
    model = LinearRegression()
    model.fit(X, y)
    
    # Forecast
    future_X = np.arange(len(df), len(df) + forecast_days).reshape(-1, 1)
    forecast = model.predict(future_X)
    
    monthly_forecast = forecast.sum()
    return {
      'forecast_daily': forecast,
      'forecast_monthly': monthly_forecast,
      'confidence': self.calculate_confidence(model, X, y)
    }
  
  def calculate_confidence(self, model, X, y):
    predictions = model.predict(X)
    residuals = y - predictions
    std_error = np.std(residuals)
    return 1 - (std_error / np.mean(y))

Key Takeaways

• **Anomaly detection** identifies unusual spending

• **Chargeback models** allocate costs to departments

• **Automated controls** enforce budgets

• **RI analysis** maximizes discount opportunities

• **Forecasting** predicts future costs

• **Granular tagging** enables cost tracking

Next Steps

Explore cost optimization recommendations from Google Cloud's cost intelligence, or learn about multi-cloud cost management strategies.