Scale, CAP, BASE &
Distributed Systems

# application.yml — Stateless session via Redis
spring:
  session:
    store-type: redis         # ← Store session in Redis, not in-memory
  data:
    redis:
      host: redis.example.com
      port: 6379

server:
  port: 8080

# Now ANY instance can handle ANY request
# Load balancer can route freely — no sticky sessions needed

@RestController
@RequestMapping("/api/orders")
public class OrderController {

    @Autowired
    private OrderService orderService;

    // ✅ STATELESS — no instance-level state
    // All state is in DB/Redis — safe for horizontal scaling
    @GetMapping("/{orderId}")
    public ResponseEntity<Order> getOrder(@PathVariable Long orderId) {
        return ResponseEntity.ok(orderService.findById(orderId));
    }

    // ❌ BAD — storing state in instance variable
    // private List<Order> cache = new ArrayList<>();
    // If ALB routes request to different instance — cache miss!
}

@Service
@Transactional(isolation = Isolation.SERIALIZABLE)  // Strongest isolation
public class BankTransferService {

    @Autowired private AccountRepository accountRepo;

    /**
     * CP Strategy: Acquire pessimistic lock on both accounts.
     * If DB is unreachable (partition), transaction fails with exception.
     * ✅ Consistent  ✅ Partition-tolerant  ❌ Unavailable during partition
     */
    public void transfer(Long fromId, Long toId, BigDecimal amount) {
        // Pessimistic lock — prevents dirty/phantom reads
        Account from = accountRepo.findByIdWithLock(fromId)
            .orElseThrow(() -> new AccountNotFoundException());
        Account to = accountRepo.findByIdWithLock(toId)
            .orElseThrow(() -> new AccountNotFoundException());

        if (from.getBalance().compareTo(amount) < 0)
            throw new InsufficientFundsException();

        from.debit(amount);
        to.credit(amount);
        accountRepo.saveAll(List.of(from, to));

        // If any step fails → entire transaction rolls back (ACID)
    }
}

// Repository with pessimistic lock
public interface AccountRepository extends JpaRepository<Account, Long> {

    @Lock(LockModeType.PESSIMISTIC_WRITE)
    @Query("SELECT a FROM Account a WHERE a.id = :id")
    Optional<Account> findByIdWithLock(@Param("id") Long id);
}

@Service
public class FeedService {

    @Autowired private FeedRepository feedRepo;      // DynamoDB backed
    @Autowired private RedisTemplate redisTemplate;   // Read cache

    /**
     * AP Strategy: Read from cache first, fall back to DB.
     * If DB is partitioned, return cached (stale) data.
     * ✅ Always available  ✅ Partition-tolerant  ❌ May serve stale posts
     */
    public List<Post> getFeed(String userId) {
        String cacheKey = "feed:" + userId;

        // Try cache first
        List<Post> cached = (List<Post>) redisTemplate.opsForValue().get(cacheKey);
        if (cached != null) return cached;

        try {
            List<Post> posts = feedRepo.getRecentPosts(userId);
            redisTemplate.opsForValue().set(cacheKey, posts, Duration.ofSeconds(30));
            return posts;
        } catch (DataAccessException e) {
            // DB unreachable during partition — return empty feed, don't fail
            // Availability over consistency — user sees no feed vs 500 error
            return Collections.emptyList();
        }
    }
}

@Service
public class OrderService {

    @Autowired private OrderRepository orderRepo;
    @Autowired private InventoryRepository inventoryRepo;
    @Autowired private PaymentRepository paymentRepo;

    /**
     * ATOMICITY: All 3 steps succeed or ALL are rolled back.
     * CONSISTENCY: Stock never goes negative (DB constraint).
     * ISOLATION: Another order cannot read the same stock simultaneously.
     * DURABILITY: After commit, data survives server crash.
     */
    @Transactional(isolation = Isolation.READ_COMMITTED,
                   propagation = Propagation.REQUIRED,
                   rollbackFor = Exception.class)
    public Order placeOrder(OrderRequest req) {
        // Step 1: Reserve inventory (Atomicity — if this fails, nothing persists)
        Inventory inv = inventoryRepo.findByProductId(req.getProductId());
        if (inv.getStock() < req.getQuantity())
            throw new InsufficientStockException(); // → triggers rollback

        inv.decrementStock(req.getQuantity());
        inventoryRepo.save(inv);

        // Step 2: Create order record
        Order order = new Order(req, OrderStatus.PENDING);
        orderRepo.save(order);

        // Step 3: Create payment record
        Payment payment = new Payment(order.getId(), req.getAmount());
        paymentRepo.save(payment);

        // ✅ All 3 steps committed atomically
        return order;
    }
}

@Service
public class ViewCounterService {

    @Autowired private DynamoDbClient dynamoDb;

    /**
     * BASE Pattern: We don't care if view count is 1,001,234 or 1,001,235.
     * Basically Available: We always accept view events.
     * Soft State: Count might be slightly off due to race conditions.
     * Eventually Consistent: Counts will converge across replicas.
     */
    public void incrementViewCount(String productId) {
        // Atomic increment — DynamoDB ADD operation
        UpdateItemRequest request = UpdateItemRequest.builder()
            .tableName("ProductStats")
            .key(Map.of("productId", AttributeValue.fromS(productId)))
            .updateExpression("ADD viewCount :inc")
            .expressionAttributeValues(Map.of(":inc", AttributeValue.fromN("1")))
            .build();

        dynamoDb.updateItem(request);
        // Fire and forget — no ACID guarantee needed here
        // Count will be eventually consistent across all read replicas
    }

    // For exact counts that matter — use Redis INCR (atomic, single thread)
    public Long getCount(String productId) {
        return redisTemplate.opsForValue().increment("views:" + productId, 0L);
    }
}

# application.yml — Spring Cloud Gateway
spring:
  cloud:
    gateway:
      routes:
        # Route 1: Extracted Product Service (new microservice)
        - id: product-service
          uri: lb://product-service        # lb:// = Eureka load balanced
          predicates:
            - Path=/api/products/**
          filters:
            - StripPrefix=1

        # Route 2: Extracted User Service
        - id: user-service
          uri: lb://user-service
          predicates:
            - Path=/api/users/**

        # Fallback: Everything else goes to monolith (still running!)
        - id: legacy-monolith
          uri: http://monolith.internal:8080
          predicates:
            - Path=/**                        # Catch-all

@Configuration
@EnableWebSecurity
public class SecurityConfig {

    @Autowired private JwtAuthFilter jwtFilter;

    @Bean
    public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
        return http
            .csrf(AbstractHttpConfigurer::disable)

            // ✅ CRITICAL: SessionCreationPolicy.STATELESS
            // Spring Security will NOT create HttpSession
            // No server-side session — safe for horizontal scaling
            .sessionManagement(sm -> sm
                .sessionCreationPolicy(SessionCreationPolicy.STATELESS))

            .authorizeHttpRequests(auth -> auth
                .requestMatchers("/api/auth/**").permitAll()
                .anyRequest().authenticated())

            // Add JWT filter before UsernamePasswordAuthenticationFilter
            .addFilterBefore(jwtFilter, UsernamePasswordAuthenticationFilter.class)
            .build();
    }
}

@Component
public class JwtAuthFilter extends OncePerRequestFilter {

    @Autowired private JwtUtil jwtUtil;

    @Override
    protected void doFilterInternal(HttpServletRequest req,
                                      HttpServletResponse res,
                                      FilterChain chain) {
        String header = req.getHeader("Authorization");

        if (header != null && header.startsWith("Bearer ")) {
            String token = header.substring(7);
            if (jwtUtil.isValid(token)) {
                // Extract claims from token — NO DB lookup needed!
                // Any instance can do this with the same secret key
                String username = jwtUtil.extractUsername(token);
                List<String> roles = jwtUtil.extractRoles(token);

                UsernamePasswordAuthenticationToken auth =
                    new UsernamePasswordAuthenticationToken(
                        username, null, mapRoles(roles));

                SecurityContextHolder.getContext().setAuthentication(auth);
            }
        }
        chain.doFilter(req, res);
    }
}

// Order Service — publishes events, doesn't call other DBs directly
@Service
public class OrderSagaOrchestrator {

    @Autowired private KafkaTemplate<String,Object> kafka;
    @Autowired private OrderRepository orderRepo;

    /**
     * SAGA: Distributed transaction without cross-service DB access
     * Step 1: Create order (PENDING) in Order DB
     * Step 2: Publish event → Payment Service consumes
     * Step 3: Payment success/fail event → Inventory Service
     * Step 4: If any step fails → compensating transaction (rollback via events)
     */
    @Transactional
    public Order createOrder(OrderRequest req) {
        Order order = new Order(req, OrderStatus.PENDING);
        orderRepo.save(order);  // Save in OWN DB only

        // Publish event — Payment Service will pick it up
        kafka.send("order.created", new OrderCreatedEvent(
            order.getId(), req.getUserId(), req.getAmount(), req.getProductId()
        ));

        return order;
    }

    // ← Listen for results from Payment Service
    @KafkaListener(topics = "payment.completed")
    public void onPaymentCompleted(PaymentCompletedEvent event) {
        Order order = orderRepo.findById(event.getOrderId()).orElseThrow();
        order.setStatus(OrderStatus.CONFIRMED);
        orderRepo.save(order);

        // Trigger next step in saga
        kafka.send("inventory.reserve", new ReserveInventoryEvent(
            order.getId(), event.getProductId(), event.getQuantity()
        ));
    }

    @KafkaListener(topics = "payment.failed")
    public void onPaymentFailed(PaymentFailedEvent event) {
        // COMPENSATING TRANSACTION — rollback order
        Order order = orderRepo.findById(event.getOrderId()).orElseThrow();
        order.setStatus(OrderStatus.CANCELLED);
        orderRepo.save(order);
        // Publish order.cancelled so other services can compensate too
        kafka.send("order.cancelled", new OrderCancelledEvent(order.getId()));
    }
}

# application.yml — Spring Cloud Gateway
spring:
  cloud:
    gateway:
      default-filters:
        - DedupeResponseHeader=Access-Control-Allow-Credentials Access-Control-Allow-Origin
        - name: RequestRateLimiter
          args:
            redis-rate-limiter.replenishRate: 10     # 10 req/sec per user
            redis-rate-limiter.burstCapacity: 20     # Allow burst of 20
            key-resolver: "#{@userKeyResolver}"

      routes:
        - id: order-service
          uri: lb://ORDER-SERVICE
          predicates:
            - Path=/api/orders/**
          filters:
            - RewritePath=/api/orders/(?<segment>.*), /orders/${segment}
            - name: CircuitBreaker           # Resilience4j circuit breaker
              args:
                name: orderServiceCB
                fallbackUri: forward:/fallback/orders

        - id: product-service
          uri: lb://PRODUCT-SERVICE
          predicates:
            - Path=/api/products/**
          filters:
            - name: Retry
              args:
                retries: 3
                statuses: BAD_GATEWAY

@Component
public class JwtGatewayFilter implements GlobalFilter, Ordered {

    @Autowired private JwtUtil jwtUtil;

    @Override
    public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
        ServerHttpRequest request = exchange.getRequest();
        String path = request.getPath().value();

        // Skip auth for public endpoints
        if (path.startsWith("/api/auth/")) return chain.filter(exchange);

        String token = extractToken(request);
        if (token == null || !jwtUtil.isValid(token)) {
            exchange.getResponse().setStatusCode(HttpStatus.UNAUTHORIZED);
            return exchange.getResponse().setComplete();
        }

        // Forward userId header to downstream services
        String userId = jwtUtil.extractUserId(token);
        ServerHttpRequest mutatedReq = request.mutate()
            .header("X-User-Id", userId)
            .header("X-User-Roles", jwtUtil.extractRoles(token).toString())
            .build();

        return chain.filter(exchange.mutate().request(mutatedReq).build());
    }

    @Override
    public int getOrder() { return -1; } // Runs before all other filters
}

// Entity: outbox_events table
@Entity
@Table(name = "outbox_events")
public class OutboxEvent {
    @Id @GeneratedValue
    private UUID id;
    private String topic;          // "order.created"
    private String aggregateId;    // orderId
    private String payload;        // JSON event body
    private OutboxStatus status;   // PENDING / PUBLISHED
    private LocalDateTime createdAt;
}

@Service
public class OrderService {
    @Autowired private OrderRepository orderRepo;
    @Autowired private OutboxRepository outboxRepo;
    @Autowired private ObjectMapper mapper;

    @Transactional  // BOTH writes in ONE DB transaction
    public Order createOrder(OrderRequest req) {
        Order order = orderRepo.save(new Order(req));

        // Write to outbox table IN SAME TRANSACTION as order
        // If Kafka is down — no problem, outbox record is safe in DB
        outboxRepo.save(new OutboxEvent("order.created",
            order.getId().toString(),
            mapper.writeValueAsString(new OrderCreatedEvent(order))));

        return order;
    }
}

// Scheduled poller publishes pending outbox events
@Component
public class OutboxPoller {
    @Autowired private OutboxRepository outboxRepo;
    @Autowired private KafkaTemplate<String,String> kafka;

    @Scheduled(fixedDelay = 1000)  // Every 1 second
    public void publishPending() {
        outboxRepo.findByStatus(OutboxStatus.PENDING).forEach(event -> {
            try {
                kafka.send(event.getTopic(), event.getAggregateId(), event.getPayload())
                     .get();  // Synchronous wait for ack
                event.setStatus(OutboxStatus.PUBLISHED);
                outboxRepo.save(event);
            } catch (Exception e) {
                // Retry on next poll — Kafka may be temporarily down
                log.warn("Failed to publish outbox event {}", event.getId());
            }
        });
    }
}

@KafkaListener(topics = "order.created", groupId = "inventory-group")
public void onOrderCreated(OrderCreatedEvent event,
                            @Header(KafkaHeaders.RECEIVED_KEY) String key) {
    // Idempotency: Check if we already processed this event
    // Multiple deliveries MUST NOT cause duplicate stock deductions
    if (processedEventRepo.existsByEventId(event.getEventId())) {
        log.info("Duplicate event {} — skipping", event.getEventId());
        return;  // Safe to ignore — already processed
    }

    inventoryService.reserveStock(event.getProductId(), event.getQuantity());

    // Mark as processed (upsert by eventId)
    processedEventRepo.save(new ProcessedEvent(event.getEventId()));
}

// Dead Letter Queue config — failed messages go here after 3 retries
@Bean
public DefaultErrorHandler kafkaErrorHandler(KafkaOperations<?, ?> template) {
    DeadLetterPublishingRecoverer dlq =
        new DeadLetterPublishingRecoverer(template,
            (r, e) -> new TopicPartition(r.topic() + ".DLQ", r.partition()));

    ExponentialBackOffWithMaxRetries backOff =
        new ExponentialBackOffWithMaxRetries(3);  // Retry 3 times
    backOff.setInitialInterval(1000L);
    backOff.setMultiplier(2.0);                  // 1s → 2s → 4s

    return new DefaultErrorHandler(dlq, backOff);
}

# application.yml
resilience4j:
  circuitbreaker:
    instances:
      payment-service:
        slidingWindowSize: 10          # Evaluate last 10 calls
        failureRateThreshold: 50        # Open if ≥50% fail
        waitDurationInOpenState: 10000  # Wait 10s before half-open
        permittedNumberOfCallsInHalfOpenState: 3
        registerHealthIndicator: true

  retry:
    instances:
      payment-service:
        maxAttempts: 3
        waitDuration: 500ms
        retryExceptions:
          - java.io.IOException
          - org.springframework.web.client.HttpServerErrorException

  ratelimiter:
    instances:
      payment-service:
        limitForPeriod: 100           # 100 calls per period
        limitRefreshPeriod: 1s
        timeoutDuration: 500ms         # Wait max 500ms for permit

@Service
public class PaymentClient {

    @Autowired private RestTemplate restTemplate;

    /**
     * Circuit Breaker states:
     * CLOSED → Normal operation, all calls go through
     * OPEN   → 50%+ failures — calls SHORT-CIRCUIT to fallback immediately
     * HALF-OPEN → Test with a few calls — if OK, go back to CLOSED
     */
    @CircuitBreaker(name = "payment-service", fallbackMethod = "paymentFallback")
    @Retry(name = "payment-service")
    @RateLimiter(name = "payment-service")
    public PaymentResponse processPayment(PaymentRequest req) {
        return restTemplate.postForObject(
            "http://payment-service/api/payments", req, PaymentResponse.class);
    }

    // Fallback — called when circuit is OPEN or all retries exhausted
    public PaymentResponse paymentFallback(PaymentRequest req, Exception e) {
        log.error("Payment service unavailable. Queuing payment: {}", req.getOrderId());

        // Graceful degradation: Queue for async processing
        kafka.send("payment.pending", req);

        return PaymentResponse.builder()
            .status(PaymentStatus.QUEUED)
            .message("Payment queued — will process shortly")
            .build();
    }
}

// Eureka Server (dedicated service)
@SpringBootApplication
@EnableEurekaServer
public class ServiceRegistryApplication {
    public static void main(String[] args) {
        SpringApplication.run(ServiceRegistryApplication.class, args);
    }
}

---

// Microservice Client Registration
@SpringBootApplication
@EnableDiscoveryClient
public class OrderServiceApplication {
    public static void main(String[] args) {
        SpringApplication.run(OrderServiceApplication.class, args);
    }

    // Load-balanced RestTemplate — lb:// prefix uses Eureka
    @Bean
    @LoadBalanced
    public RestTemplate restTemplate() { return new RestTemplate(); }
}

// Calling another service by name (not hardcoded URL!)
@Service
public class OrderServiceImpl {
    @Autowired private RestTemplate restTemplate;  // @LoadBalanced

    public Product getProduct(Long productId) {
        // Eureka resolves "PRODUCT-SERVICE" to actual IP/port
        // Handles multiple instances with load balancing
        return restTemplate.getForObject(
            "http://PRODUCT-SERVICE/api/products/" + productId,
            Product.class);
    }
}

dependencies:
  - micrometer-tracing-bridge-brave
  - zipkin-reporter-brave

spring:
  application:
    name: order-service         # Appears in Zipkin traces
  zipkin:
    base-url: http://zipkin:9411

management:
  tracing:
    sampling:
      probability: 1.0           # 100% in dev; use 0.1 in prod

# Trace propagates automatically across REST + Kafka calls
# Trace ID threads through: Gateway → Order → Payment → Inventory
# Find slowest service easily in Zipkin UI

@Configuration
public class KafkaProducerConfig {

    @Bean
    public ProducerFactory<String, Object> producerFactory() {
        Map<String, Object> props = new HashMap<>();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka:9092");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);

        // Idempotent producer — exactly-once semantics at Kafka level
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true);

        // acks=all: wait for all replicas to acknowledge
        props.put(ProducerConfig.ACKS_CONFIG, "all");

        // Retry with backoff
        props.put(ProducerConfig.RETRIES_CONFIG, 3);
        props.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 1000);

        return new DefaultKafkaProducerFactory<>(props);
    }
}

// Publishing with custom partition key for ordering guarantee
@Service
public class EventPublisher {
    @Autowired private KafkaTemplate<String, Object> kafka;

    public void publishOrderEvent(Order order) {
        OrderCreatedEvent event = new OrderCreatedEvent(order);

        // KEY = orderId → all events for same order go to SAME PARTITION
        // Guarantees ordering: CREATED → PAID → SHIPPED → DELIVERED
        kafka.send("order.events", order.getId().toString(), event)
             .whenComplete((result, ex) -> {
                 if (ex != null) log.error("Failed to publish event", ex);
                 else log.info("Published to partition {}",
                     result.getRecordMetadata().partition());
             });
    }
}

# ────────────────────────────────────────
# STAGE 1: Build
# ────────────────────────────────────────
FROM eclipse-temurin:21-jdk-alpine AS builder
WORKDIR /workspace

# Copy Maven wrapper and pom.xml first (layer caching for dependencies)
COPY mvnw .
COPY .mvn .mvn
COPY pom.xml .

# Download dependencies (cached layer — only re-runs if pom.xml changes)
RUN ./mvnw dependency:go-offline -q

# Copy source and build
COPY src src
RUN ./mvnw package -DskipTests -q

# ────────────────────────────────────────
# STAGE 2: Extract layers (Spring Boot 3 layered jar)
# ────────────────────────────────────────
FROM eclipse-temurin:21-jre-alpine AS extractor
WORKDIR /workspace
COPY --from=builder /workspace/target/*.jar app.jar
RUN java -Djarmode=layertools -jar app.jar extract

# ────────────────────────────────────────
# STAGE 3: Runtime (smallest possible image)
# ────────────────────────────────────────
FROM eclipse-temurin:21-jre-alpine
WORKDIR /app

# Non-root user for security
RUN addgroup -S spring && adduser -S spring -G spring
USER spring:spring

# Layered copy — only changed layers re-download on deploy
COPY --from=extractor /workspace/dependencies/ ./
COPY --from=extractor /workspace/spring-boot-loader/ ./
COPY --from=extractor /workspace/snapshot-dependencies/ ./
COPY --from=extractor /workspace/application/ ./

EXPOSE 8080

# JVM tuning for containers
ENV JAVA_OPTS="-XX:MaxRAMPercentage=75.0 -XX:+UseContainerSupport -XX:+UseG1GC"
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS org.springframework.boot.loader.launch.JarLauncher"]

version: '3.9'

services:

  # ── Infrastructure ──────────────────────────────────────

  postgres:
    image: postgres:16-alpine
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: secret
      POSTGRES_DB: ordersdb
    ports: ["5432:5432"]
    volumes: [postgres_data:/var/lib/postgresql/data]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U admin"]
      interval: 5s

  redis:
    image: redis:7-alpine
    ports: ["6379:6379"]
    command: redis-server --appendonly yes

  kafka:
    image: confluentinc/cp-kafka:7.6.0
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on: [zookeeper]
    ports: ["9092:9092"]

  zookeeper:
    image: confluentinc/cp-zookeeper:7.6.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181

  # ── Services ─────────────────────────────────────────────

  api-gateway:
    build: ./api-gateway
    ports: ["8080:8080"]
    environment:
      EUREKA_URI: http://service-registry:8761/eureka
    depends_on: [service-registry]

  order-service:
    build: ./order-service
    environment:
      SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/ordersdb
      SPRING_KAFKA_BOOTSTRAP_SERVERS: kafka:9092
      EUREKA_URI: http://service-registry:8761/eureka
    depends_on:
      postgres: { condition: service_healthy }
      kafka: { condition: service_started }
    deploy:
      replicas: 2                    # 2 instances for local HA test

  service-registry:
    build: ./service-registry
    ports: ["8761:8761"]

volumes:
  postgres_data:

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  namespace: production
  labels:
    app: order-service
    version: v2.1.0
spec:
  replicas: 3
  selector:
    matchLabels:
      app: order-service
  strategy:
    type: RollingUpdate            # Zero-downtime deploys
    rollingUpdate:
      maxUnavailable: 1           # At most 1 Pod down during update
      maxSurge: 1                 # At most 1 extra Pod during update
  template:
    metadata:
      labels:
        app: order-service
    spec:
      containers:
        - name: order-service
          image: myregistry.ecr.aws/order-service:v2.1.0
          ports:
            - containerPort: 8080

          # Resource limits — prevents one Pod from starving others
          resources:
            requests:
              memory: "256Mi"
              cpu: "250m"           # 0.25 CPU core
            limits:
              memory: "512Mi"
              cpu: "500m"

          # Spring Boot Actuator health probes
          livenessProbe:           # K8s restarts Pod if this fails
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            initialDelaySeconds: 60
            periodSeconds: 10

          readinessProbe:          # K8s stops sending traffic until ready
            httpGet:
              path: /actuator/health/readiness
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 5

          # Config from ConfigMap + Secrets
          envFrom:
            - configMapRef:
                name: order-service-config
            - secretRef:
                name: order-service-secrets

---
apiVersion: v1
kind: Service
metadata:
  name: order-service
  namespace: production
spec:
  selector:
    app: order-service
  ports:
    - port: 80
      targetPort: 8080
  type: ClusterIP                   # Internal only — Gateway routes to this

---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: order-service-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  minReplicas: 3
  maxReplicas: 20                    # Auto-scales up to 20 Pods
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 60    # Scale up if CPU >60%

# application.yml
management:
  endpoint:
    health:
      probes:
        enabled: true            # /actuator/health/liveness + /readiness
      show-details: always
  health:
    livenessstate:
      enabled: true
    readinessstate:
      enabled: true
    db:
      enabled: true             # Checks DB connection
    redis:
      enabled: true             # Checks Redis connection
    kafka:
      enabled: true             # Checks Kafka connection

---

// Custom health indicator for downstream dependency
@Component
public class PaymentServiceHealthIndicator implements HealthIndicator {

    @Autowired private PaymentClient paymentClient;

    @Override
    public Health health() {
        try {
            paymentClient.ping();    // Quick health check call
            return Health.up()
                .withDetail("status", "Payment service reachable")
                .build();
        } catch (Exception e) {
            // If payment service is down, mark this pod as NOT READY
            // K8s will stop routing traffic here until it recovers
            return Health.down()
                .withDetail("error", e.getMessage())
                .build();
        }
    }
}

# pom.xml dependency
<dependency>
    <groupId>io.awspring.cloud</groupId>
    <artifactId>spring-cloud-aws-secrets-manager-config</artifactId>
</dependency>

---

# bootstrap.yml — loads BEFORE application context
spring:
  cloud:
    aws:
      secretsmanager:
        enabled: true
        region: ap-south-1          # Mumbai region for India
      credentials:
        instance-profile: true       # Use EKS Pod IAM role (IRSA)

# Secret name in AWS: /prod/order-service/db
# Contains JSON: { "username": "admin", "password": "super-secret" }
# Spring automatically maps to spring.datasource.username / .password

---

// IRSA (IAM Roles for Service Accounts) — no hardcoded AWS keys in pods!
// Kubernetes ServiceAccount → IAM Role → Secrets Manager access
// This is the production-secure way. No AWS_ACCESS_KEY_ID needed.

// Annotate Kubernetes ServiceAccount:
// annotations:
//   eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT_ID:role/order-service-role

name: Build and Deploy to EKS

on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # Step 1: Build Spring Boot JAR
      - name: Build JAR
        run: mvn clean package -DskipTests

      # Step 2: Configure AWS credentials (OIDC — no secrets stored!)
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: ap-south-1

      # Step 3: Push image to ECR
      - name: Login to ECR
        run: aws ecr get-login-password | docker login --username AWS --password-stdin $ECR_URI

      - name: Build & Push Docker Image
        run: |
          docker build -t $ECR_URI/order-service:${{ github.sha }} .
          docker push $ECR_URI/order-service:${{ github.sha }}

      # Step 4: Deploy to EKS
      - name: Update kubeconfig
        run: aws eks update-kubeconfig --name prod-cluster --region ap-south-1

      - name: Deploy (Rolling Update)
        run: |
          kubectl set image deployment/order-service \
            order-service=$ECR_URI/order-service:${{ github.sha }} \
            -n production

          # Wait for rollout to complete before marking deploy success
          kubectl rollout status deployment/order-service -n production