Seeing the Invisible: Building AI Observability That Tells the Whole Story

Request → Function → Database → Response

Request → Intent Classification → Context Loading →
Model Inference → Safety Checks → Response Generation →
Audit Logging → Delivery

User: "Cancel my subscription"
AI: [some response]
Engineer: "Did it work?"
Team: "¯\_(ツ)_/¯"

User: "Cancel my subscription"

Telemetry:
- Intent: CancelSubscription (confidence: 0.94)
- Model: bedrock-claude-v2 (primary)
- Context: 2,340 tokens loaded
- Policy Check: Requires manager approval
- Guardrails: 0 triggered
- Latency: 680ms
- Cost: $0.042
- Outcome: Approval workflow created
- User Satisfaction: 9/10 (post-interaction survey)

300">interface ComplianceEvent {
  eventId: string
  timestamp: number
  userId: string
  brand: string
  action: string
  intent?: string

  // Data access
  dataAccessed: {
    types: string[]          // ["customer_name", "billing_info"]
    classification: string   // "PII", "PHI", "financial"
    justification: string    // Why this data was needed
  }

  // AI-specific
  aiMetadata?: {
    modelUsed: string
    inputTokens: number
    outputTokens: number
    piiDetected: boolean
    piiRedacted: boolean
    guardrailsTriggered: string[]
  }

  // Compliance
  complianceFrameworks: string[]  // ["SOC2", "GDPR", "HIPAA"]
  retentionPolicy: string

  // Audit
  ipAddress: string
  userAgent: string
  sessionId: string
}

{
  "eventId": "evt_20250122_abc123",
  "timestamp": 1705910400000,
  "userId": "user_789",
  "brand": "securitain",
  "action": "ai_query_processed",
  "intent": "CheckComplianceStatus",

  "dataAccessed": {
    "types": ["company_name", "compliance_framework", "audit_logs"],
    "classification": "business_confidential",
    "justification": "User requested compliance status for their organization"
  },

  "aiMetadata": {
    "modelUsed": "bedrock-claude-v2",
    "inputTokens": 1240,
    "outputTokens": 580,
    "piiDetected": 300">false,
    "piiRedacted": 300">false,
    "guardrailsTriggered": []
  },

  "complianceFrameworks": ["SOC2", "GDPR"],
  "retentionPolicy": "7_years",

  "ipAddress": "203.0.113.42",
  "userAgent": "Mozilla/5.0...",
  "sessionId": "sess_xyz789"
}

// Generate compliance report
300">const report = 300">await CoreCloud.generateComplianceReport({
  startDate: '2024-01-01',
  endDate: '2024-12-31',
  framework: 'SOC2',
  brand: 'securitain'
})

// Report includes:
// - Total AI interactions: 2.4M
// - PII access events: 847K (all authorized)
// - Guardrail triggers: 1,203 (all handled correctly)
// - Unauthorized access attempts: 0
// - Data retention compliance: 100%
// - Encryption compliance: 100%

// Alert on suspicious activity
300">await CloudWatch.putMetricAlarm({
  AlarmName: 'UnauthorizedAIDataAccess',
  MetricName: 'UnauthorizedAccess',
  Namespace: 'Cloudain/Compliance',
  Threshold: 1,
  ComparisonOperator: 'GreaterThanOrEqualToThreshold',
  AlarmActions: [
    securityTeamSNS,
    complianceTeamSNS,
    autoBlockUserLambda
  ]
})

300">interface AITelemetry {
  // Request metadata
  requestId: string
  timestamp: number
  brand: string
  userId: string
  sessionId: string

  // Intent analysis
  intent: {
    detected: string
    confidence: number
    alternativeIntents: Array<{
      name: string
      confidence: number
    }>
    classificationLatency: number
  }

  // Context & memory
  context: {
    tokensLoaded: number
    messagesInContext: number
    cacheHit: boolean
    contextLoadLatency: number
  }

  // Model inference
  model: {
    provider: string
    modelId: string
    parameters: {
      temperature: number
      maxTokens: number
      topP?: number
    }
    inputTokens: number
    outputTokens: number
    inferenceLatency: number
    cost: number
    retryCount: number
    fallbackUsed: boolean
  }

  // Safety & compliance
  safety: {
    piiDetected: boolean
    piiRedacted: boolean
    guardrailsChecked: string[]
    guardrailsTriggered: string[]
    contentFiltered: boolean
  }

  // Response quality
  response: {
    completed: boolean
    truncated: boolean
    characterCount: number
    sentimentScore?: number
  }

  // Performance
  performance: {
    totalLatency: number
    breakdown: {
      auth: number
      intentClassification: number
      contextLoading: number
      modelInference: number
      safetyChecks: number
      responseFormatting: number
    }
  }

  // Business metrics
  business: {
    resolved: boolean
    escalated: boolean
    userSatisfaction?: number
    followUpRequired: boolean
  }

  // Errors
  errors?: Array<{
    300">type: string
    message: string
    timestamp: number
    recovered: boolean
  }>
}

// Stream telemetry to Kinesis
300">async 300">function emitTelemetry(telemetry: AITelemetry) {
  300">await Kinesis.putRecord({
    StreamName: 'cloudain-ai-telemetry',
    PartitionKey: telemetry.brand,
    Data: JSON.stringify(telemetry)
  })

  // Also log to CloudWatch for real-time dashboards
  300">await CloudWatch.putMetricData({
    Namespace: 'Cloudain/AgenticCloud',
    MetricData: [
      {
        MetricName: 'AILatency',
        Value: telemetry.performance.totalLatency,
        Unit: 'Milliseconds',
        Dimensions: [
          { Name: 'Brand', Value: telemetry.brand },
          { Name: 'Intent', Value: telemetry.intent.detected }
        ]
      },
      {
        MetricName: 'TokenUsage',
        Value: telemetry.model.inputTokens + telemetry.model.outputTokens,
        Unit: 'Count',
        Dimensions: [
          { Name: 'Brand', Value: telemetry.brand },
          { Name: 'Model', Value: telemetry.model.modelId }
        ]
      },
      {
        MetricName: 'AIFost',
        Value: telemetry.model.cost,
        Unit: 'None',
        Dimensions: [
          { Name: 'Brand', Value: telemetry.brand }
        ]
      }
    ]
  })
}

// CloudWatch Dashboard
300">const dashboard = {
  widgets: [
    // Total Conversations
    {
      300">type: 'metric',
      properties: {
        metrics: [
          ['Cloudain/AgenticCloud', 'Conversations', { stat: 'Sum' }]
        ],
        period: 3600,
        stat: 'Sum',
        region: 'us-east-1',
        title: 'Conversations per Hour'
      }
    },

    // Average Latency by Brand
    {
      300">type: 'metric',
      properties: {
        metrics: [
          ['...', { stat: 'Average', label: 'Growain' }],
          ['...', { stat: 'Average', label: 'Securitain' }],
          ['...', { stat: 'Average', label: 'MindAgain' }]
        ],
        title: 'Average Latency by Brand',
        yAxis: { left: { min: 0, max: 2000 }}
      }
    },

    // Cost Tracking
    {
      300">type: 'metric',
      properties: {
        metrics: [
          ['Cloudain/AgenticCloud', 'AICost', { stat: 'Sum' }]
        ],
        title: 'AI Cost (Last 24 Hours)',
        stat: 'Sum',
        period: 86400
      }
    },

    // User Satisfaction
    {
      300">type: 'metric',
      properties: {
        metrics: [
          ['Cloudain/Business', 'UserSatisfaction', { stat: 'Average' }]
        ],
        title: 'Average User Satisfaction (1-10)',
        yAxis: { left: { min: 0, max: 10 }}
      }
    }
  ]
}

// Latency percentiles
300">const latencyMetrics = {
  widget: {
    300">type: 'metric',
    properties: {
      metrics: [
        ['Cloudain/AgenticCloud', 'AILatency', { stat: 'p50' }],
        ['...', { stat: 'p95' }],
        ['...', { stat: 'p99' }]
      ],
      title: 'Latency Percentiles',
      period: 300
    }
  }
}

// Error tracking
300">const errorMetrics = {
  widget: {
    300">type: 'metric',
    properties: {
      metrics: [
        ['Cloudain/AgenticCloud', 'Errors', { stat: 'Sum' }],
        ['...', 'IntentClassificationErrors', { stat: 'Sum' }],
        ['...', 'ModelInferenceErrors', { stat: 'Sum' }],
        ['...', 'ContextLoadErrors', { stat: 'Sum' }]
      ],
      title: 'Error Breakdown',
      period: 300
    }
  }
}

// Compliance widget
300">const complianceWidget = {
  widget: {
    300">type: 'log',
    properties: {
      query: `
        SOURCE 'cloudain-audit-logs'
        | fields @timestamp, userId, action, dataAccessed
        | filter action = 'pii_accessed'
        | stats count() by userId
        | sort count desc
      `,
      title: 'PII Access by User (Last 24h)',
      region: 'us-east-1'
    }
  }
}

// Guardrail monitoring
300">const guardrailWidget = {
  widget: {
    300">type: 'metric',
    properties: {
      metrics: [
        ['Cloudain/Safety', 'GuardrailTriggered', { stat: 'Sum' }]
      ],
      title: 'Safety Guardrail Activations',
      annotations: {
        horizontal: [{
          value: 100,
          label: 'Review Threshold'
        }]
      }
    }
  }
}

// Instrument AI request flow
300">import AWSXRay 300">from 'aws-xray-sdk-core'

300">async 300">function processAIRequest(request: Request) {
  300">const segment = AWSXRay.getSegment()

  // Intent classification subsegment
  300">const intentSegment = segment.addNewSubsegment('IntentClassification')
  intentSegment.addAnnotation('brand', request.brand)

  300">const intent = 300">await classifyIntent(request.message)

  intentSegment.addMetadata('result', {
    intent: intent.name,
    confidence: intent.confidence
  })
  intentSegment.close()

  // Context loading subsegment
  300">const contextSegment = segment.addNewSubsegment('ContextLoading')
  300">const context = 300">await loadContext(request.sessionId)
  contextSegment.addMetadata('tokensLoaded', context.tokens)
  contextSegment.close()

  // Model inference subsegment
  300">const modelSegment = segment.addNewSubsegment('ModelInference')
  modelSegment.addAnnotation('model', 'bedrock-claude-v2')

  300">const response = 300">await generateResponse(intent, context)

  modelSegment.addMetadata('tokens', {
    input: response.inputTokens,
    output: response.outputTokens,
    cost: response.cost
  })
  modelSegment.close()

  300">return response
}

Request (850ms total)
├─ Auth Verification (15ms)
├─ Intent Classification (120ms)
│  ├─ Load model (45ms)
│  └─ Inference (75ms)
├─ Context Loading (85ms)
│  ├─ Redis lookup (5ms)
│  └─ DynamoDB query (80ms)
├─ Model Inference (580ms) ← Bottleneck
│  ├─ Token encoding (20ms)
│  ├─ Bedrock API call (540ms)
│  └─ Response parsing (20ms)
├─ Safety Checks (35ms)
└─ Response Formatting (15ms)

// Alert on latency degradation
300">await CloudWatch.putMetricAlarm({
  AlarmName: 'AILatencyDegradation',
  ComparisonOperator: 'GreaterThanThreshold',
  EvaluationPeriods: 2,
  MetricName: 'AILatency',
  Namespace: 'Cloudain/AgenticCloud',
  Period: 300,
  Statistic: 'Average',
  Threshold: 1000, // 1 second
  ActionsEnabled: 300">true,
  AlarmActions: [engineeringSNS],
  AlarmDescription: 'AI response time exceeds 1 second'
})

// Alert on cost spike
300">await CloudWatch.putMetricAlarm({
  AlarmName: 'AICostSpike',
  ComparisonOperator: 'GreaterThanThreshold',
  EvaluationPeriods: 1,
  MetricName: 'AICost',
  Namespace: 'Cloudain/AgenticCloud',
  Period: 3600,
  Statistic: 'Sum',
  Threshold: 500, // $500/hour
  ActionsEnabled: 300">true,
  AlarmActions: [
    financeTeamSNS,
    autoThrottleLambda
  ]
})

// Alert on guardrail triggers
300">await CloudWatch.putMetricAlarm({
  AlarmName: 'HighGuardrailActivity',
  ComparisonOperator: 'GreaterThanThreshold',
  EvaluationPeriods: 1,
  MetricName: 'GuardrailTriggered',
  Namespace: 'Cloudain/Safety',
  Period: 300,
  Statistic: 'Sum',
  Threshold: 50,
  ActionsEnabled: 300">true,
  AlarmActions: [securityTeamSNS]
})

// ML-powered anomaly detection
300">const anomalyDetector = {
  MetricName: 'AILatency',
  Namespace: 'Cloudain/AgenticCloud',
  Stat: 'Average',
  Dimensions: [{ Name: 'Brand', Value: 'growain' }]
}

300">await CloudWatch.putAnomalyDetector({
  ...anomalyDetector,
  Configuration: {
    ExcludedTimeRanges: [
      {
        StartTime: 300">new Date('2024-12-25T00:00:00Z'),
        EndTime: 300">new Date('2024-12-26T00:00:00Z')
      }
    ]
  }
})

// Alert on anomalies
300">await CloudWatch.putMetricAlarm({
  AlarmName: 'AILatencyAnomaly',
  ComparisonOperator: 'LessThanLowerOrGreaterThanUpperThreshold',
  EvaluationPeriods: 2,
  Metrics: [
    {
      Id: 'm1',
      ReturnData: 300">true,
      MetricStat: {
        Metric: anomalyDetector,
        Period: 300,
        Stat: 'Average'
      }
    },
    {
      Id: 'ad1',
      Expression: 'ANOMALY_DETECTION_BAND(m1, 2)', // 2 std deviations
      Label: 'AILatency (expected)'
    }
  ],
  ThresholdMetricId: 'ad1',
  AlarmActions: [engineeringSNS]
})

// Structured log format
300">const logger = winston.createLogger({
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.json()
  ),
  transports: [
    300">new winston.transports.Console(),
    300">new CloudWatchTransport({
      logGroupName: 'cloudain-ai-logs',
      logStreamName: `${brand}-${environment}`
    })
  ]
})

// Log AI interaction
logger.info('AI interaction completed', {
  requestId: request.id,
  userId: user.id,
  brand: 'growain',
  intent: 'CampaignAnalysis',
  latency: 680,
  tokens: { input: 1200, output: 350 },
  cost: 0.042,
  satisfied: 300">true
})

-- Top 10 slowest intents
fields @timestamp, intent, latency
| filter brand = "growain"
| stats avg(latency) as avg_latency by intent
| sort avg_latency desc
| limit 10

-- Cost by brand (last 24 hours)
fields @timestamp, brand, cost
| stats sum(cost) as total_cost by brand
| sort total_cost desc

-- Error analysis
fields @timestamp, error.300">type, error.message, brand, intent
| filter ispresent(error)
| stats count() as error_count by error.300">type, brand
| sort error_count desc

-- Guardrail triggers
fields @timestamp, userId, guardrail, intent
| filter guardrailsTriggered.0 != ""
| stats count() by guardrail
| sort count desc

// Stream telemetry to S3 for analytics
Kinesis → Firehose → S3 → Glue → Athena/Redshift

// Firehose configuration
300">const firehoseConfig = {
  DeliveryStreamName: 'cloudain-ai-analytics',
  S3DestinationConfiguration: {
    BucketARN: 'arn:aws:s3:::cloudain-analytics',
    Prefix: 'ai-telemetry/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/',
    BufferingHints: {
      SizeInMBs: 128,
      IntervalInSeconds: 300
    },
    CompressionFormat: 'GZIP',
    DataFormatConversionConfiguration: {
      Enabled: 300">true,
      SchemaConfiguration: {
        DatabaseName: 'cloudain_analytics',
        TableName: 'ai_telemetry',
        Region: 'us-east-1',
        CatalogId: AWS_ACCOUNT_ID
      },
      OutputFormatConfiguration: {
        Serializer: {
          ParquetSerDe: {}
        }
      }
    }
  }
}

-- User engagement patterns
SELECT
  brand,
  intent,
  COUNT(*) as interaction_count,
  AVG(user_satisfaction) as avg_satisfaction,
  AVG(total_latency) as avg_latency_ms,
  SUM(cost) as total_cost
FROM ai_telemetry
WHERE date >= CURRENT_DATE - INTERVAL '30' DAY
GROUP BY brand, intent
ORDER BY interaction_count DESC

-- Model performance comparison
SELECT
  model_id,
  AVG(inference_latency) as avg_latency,
  AVG(output_tokens) as avg_output_length,
  AVG(cost) as avg_cost,
  SUM(CASE WHEN errors IS NOT NULL THEN 1 ELSE 0 END) as error_count
FROM ai_telemetry
WHERE date = CURRENT_DATE
GROUP BY model_id

-- Cost trends
SELECT
  DATE_TRUNC('hour', timestamp) as hour,
  brand,
  SUM(cost) as hourly_cost,
  COUNT(*) as conversation_count,
  SUM(cost) / COUNT(*) as cost_per_conversation
FROM ai_telemetry
WHERE date >= CURRENT_DATE - INTERVAL '7' DAY
GROUP BY hour, brand
ORDER BY hour DESC

// Redact PII before logging
300">async 300">function logWithRedaction(event: any) {
  300">const redacted = 300">await redactPII(event)

  logger.info('AI interaction', redacted)

  // Store PII mapping separately (encrypted, short TTL)
  300">if (event.containsPII) {
    300">await CoreCloud.storePIIMapping({
      requestId: event.requestId,
      mapping: event.piiMapping,
      ttl: 300 // 5 minutes
    })
  }
}

// Don't log individual user queries
// ✗ BAD:
logger.info('User asked: "What is my credit card number?"')

// ✓ GOOD:
logger.info('Intent classified', {
  intent: 'AccountInquiry',
  confidence: 0.89,
  piiDetected: 300">true,
  piiRedacted: 300">true
  // No actual query text logged
})