data-pipeline-job.yaml

apiVersion: batch/v1
kind: CronJob
metadata:
  name: data-pipeline-sequence
spec:
  schedule: "0 0 31 2 *"  # This schedule will effectively never run automatically
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          volumes:
          - name: shared-data
            emptyDir: {}
          initContainers:
          - name: extract
            image: us-docker.pkg.dev/${PROJECT_ID}/my-docker-repo/data-pipeline-extract:latest
            envFrom:
            - secretRef:
                name: db-credentials
            volumeMounts:
            - name: shared-data
              mountPath: /data
          - name: load
            image: us-docker.pkg.dev/${PROJECT_ID}/my-docker-repo/data-pipeline-load:latest
            envFrom:
            - secretRef:
                name: db-credentials
            volumeMounts:
            - name: shared-data
              mountPath: /data
          containers:
          - name: transform
            image: us-docker.pkg.dev/${PROJECT_ID}/my-docker-repo/data-pipeline-transform:latest
            envFrom:
            - secretRef:
                name: db-credentials
            volumeMounts:
            - name: shared-data
              mountPath: /data