Deploy Feast on Kubernetes

Architecture (Click to Enlarge)

minikube start --cpus=4 --memory=7000 --driver=docker

Build Feast Docker Image Locally

go to feast/ folder. 就會看到以下檔案

project: test
registry: # https://docs.feast.dev/reference/registries/sql
  registry_type: sql
  path: postgresql+psycopg://postgres:password@registry.feast.svc.cluster.local:5432
  cache_ttl_seconds: 60
  sqlalchemy_config_kwargs:
    echo: false
    pool_pre_ping: true
entity_key_serialization_version: 3
provider: local 
online_store: # https://docs.feast.dev/reference/online-stores/redis
  type: redis
  connection_string: "online-store.feast.svc.cluster.local:6379"
offline_store: # https://docs.feast.dev/reference/offline-stores/bigquery
  type: bigquery
  dataset: feast
  project_id: mlops-437709
  billing_project_id: mlops-437709
  location: US

• registry: PostgreSQL
• online_store: Redis
• offline_store: BigQuery

account_features.py

from datetime import timedelta

from feast import (BigQuerySource, Entity, FeatureService, FeatureView,
                   ValueType, Field)
from feast.types import String, Int64, Bool

# Data Sources
# https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.bigquery_source.BigQuerySource
ds_acct_fraud_7d = BigQuerySource(
    table=f"mlops-437709.dbt_kclai.feat_acct_fraud_7d",
    timestamp_field="feature_timestamp"
)

ds_acct_num_txns_7d = BigQuerySource(
    table=f"mlops-437709.dbt_kclai.feat_acct_num_txns_7d",
    timestamp_field="feature_timestamp"
)

ds_acct_profiles = BigQuerySource(
    table=f"mlops-437709.dbt_kclai.feat_acct_profiles",
    timestamp_field="feature_timestamp"
)

# Entity
account_entity = Entity(
    name="Account",
    description="A user that has executed a transaction or received a transaction",
    value_type=ValueType.STRING,
    join_keys=["entity_id"]
)

# Feature Views
fv_acct_fraud_7d = FeatureView(
    name="acct_fraud_7d",
    entities=[account_entity],
    schema=[
        Field(name="has_fraud_7d", dtype=Bool)
    ],
    ttl=timedelta(weeks=52),
    source=ds_acct_fraud_7d
)


fv_acct_num_txns_7d = FeatureView(
    name="acct_num_txns_7d",
    entities=[account_entity],
    schema=[
        Field(name="num_transactions_7d", dtype=Int64)
    ],
    ttl=timedelta(weeks=1),
    source=ds_acct_num_txns_7d
)

fv_acct_profiles = FeatureView(
    name="acct_profiles",
    entities=[account_entity],
    schema=[
        Field(name="credit_score", dtype=Int64),
        Field(name="account_age_days", dtype=Int64),
        Field(name="has_2fa_installed", dtype=Bool)
    ],
    ttl=timedelta(weeks=52),
    source=ds_acct_profiles
)



# Feature Services
# Versioning features that power ML models:
# https://docs.feast.dev/master/how-to-guides/running-feast-in-production#id-3.2-versioning-features-that-power-ml-models
fs_fraud_detection_v1 = FeatureService(
    name="fraud_detection_v1",
    features=[
        fv_acct_fraud_7d,
        fv_acct_num_txns_7d[["num_transactions_7d"]],
        fv_acct_profiles
    ]
)

requirements.txt

feast[gcp,redis]==0.49.0
psycopg[binary]==3.2.7

entrypoint.sh

#!/bin/bash
set -e

echo "Running feast apply..."
feast apply

echo "Starting feast server..."
exec feast serve --host 0.0.0.0 --port 8080

Dockerfile

# Use Python 3.10 slim as the base image
FROM python:3.10-slim

# Set the working directory
WORKDIR /app

# Install Python dependencies
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
COPY entrypoint.sh /app/entrypoint.sh
RUN chmod +x entrypoint.sh

COPY feature_store.yaml /app/feature_store.yaml
COPY account_features.py /app/account_features.py

# Set the entrypoint to run Feast
ENTRYPOINT ["./entrypoint.sh"]

Build image

docker buildx build \
  --platform linux/amd64 \
  -t feast:v0.1.0 \
  -t feast:latest \
  .

確認建置成功

docker images --filter=reference="feast*"

REPOSITORY   TAG       IMAGE ID       CREATED          SIZE
feast        v0.1.0    436abcf371e9   55 seconds ago   596MB

Load Image into Minikube Cluster

在Local進入minikube

minikube ssh

查看目前minikube裡有的images，沒有feast

docker@minikube:~$ docker images

REPOSITORY                                TAG        IMAGE ID       CREATED         SIZE
registry.k8s.io/kube-apiserver            v1.30.0    181f57fd3cdb   12 months ago   112MB
registry.k8s.io/kube-controller-manager   v1.30.0    68feac521c0f   12 months ago   107MB
registry.k8s.io/kube-proxy                v1.30.0    cb7eac0b42cc   12 months ago   87.9MB
registry.k8s.io/kube-scheduler            v1.30.0    547adae34140   12 months ago   60.5MB
registry.k8s.io/etcd                      3.5.12-0   014faa467e29   15 months ago   139MB
registry.k8s.io/coredns/coredns           v1.11.1    2437cf762177   21 months ago   57.4MB
registry.k8s.io/pause                     3.9        829e9de338bd   2 years ago     514kB
gcr.io/k8s-minikube/storage-provisioner   v5         ba04bb24b957   4 years ago     29MB
docker@minikube:~$ 

在local開啟另個terminal，將local的docker image載入到minikube cluster裡

minikube image load feast:v0.1.0

回到minikube裡，查看images，有feast了

docker@minikube:~$ docker images

REPOSITORY                                TAG        IMAGE ID       CREATED          SIZE
feast                                     v0.1.0     436abcf371e9   33 minutes ago   596MB
registry.k8s.io/kube-apiserver            v1.30.0    181f57fd3cdb   12 months ago    112MB
registry.k8s.io/kube-controller-manager   v1.30.0    68feac521c0f   12 months ago    107MB
registry.k8s.io/kube-scheduler            v1.30.0    547adae34140   12 months ago    60.5MB
registry.k8s.io/kube-proxy                v1.30.0    cb7eac0b42cc   12 months ago    87.9MB
registry.k8s.io/etcd                      3.5.12-0   014faa467e29   15 months ago    139MB
registry.k8s.io/coredns/coredns           v1.11.1    2437cf762177   21 months ago    57.4MB
registry.k8s.io/pause                     3.9        829e9de338bd   2 years ago      514kB
gcr.io/k8s-minikube/storage-provisioner   v5         ba04bb24b957   4 years ago      29MB

K8S

Registry

registry.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: registry
  namespace: feast
spec:
  replicas: 1
  selector:
    matchLabels:
      app: registry
  template:
    metadata:
      labels:
        app: registry
    spec:
      containers:
        - name: registry
          image: postgres
          env:
            - name: POSTGRES_DB
              value: feast
            - name: POSTGRES_USER
              value: postgres
            - name: POSTGRES_PASSWORD
              value: password
          ports:
            - containerPort: 5432
          volumeMounts:
            - name: storage
              mountPath: /var/lib/postgresql/data
      volumes:
        - name: storage
          hostPath:
            path: /home/docker/data/feast/registry
            type: DirectoryOrCreate          
---
apiVersion: v1
kind: Service
metadata:
  name: registry
  namespace: feast
spec:
  selector:
    app: registry
  type: ClusterIP
  ports:
    - port: 5432
      targetPort: 5432

Online Store

online-store.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: online-store
  namespace: feast
spec:
  replicas: 1
  selector:
    matchLabels:
      app: online-store
  template:
    metadata:
      labels:
        app: online-store
    spec:
      containers:
        - name: online-store
          image: redis
          ports:
            - containerPort: 6379
              protocol: TCP
          volumeMounts:
            - name: storage
              mountPath: /data
      restartPolicy: Always
      volumes:
        - name: storage
          hostPath:
            path: /home/docker/data/feast/online-store
            type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
  name: online-store
  namespace: feast
spec:
  selector:
    app: online-store
  type: ClusterIP
  ports:
    - port: 6379
      targetPort: 6379

Secret

Create GCP Service Account Key

確認Project id

gcloud config get-value project

mlops-437709

建立Service Account

gcloud iam service-accounts create feast-sa \
    --display-name "Feast Service Account"

Reauthentication required.
Please enter your password:
Reauthentication successful.
Created service account [feast-sa].

替Service Account加上BigQuery權限

gcloud projects add-iam-policy-binding mlops-437709 \
    --member="serviceAccount:feast-sa@mlops-437709.iam.gserviceaccount.com" \
    --role="roles/bigquery.admin"

 [1] EXPRESSION=request.time < timestamp("2025-04-09T07:42:05.596Z"), TITLE=cloudbuild-connection-setup
 [2] None
 [3] Specify a new condition
The policy contains bindings with conditions, so specifying a condition is required when adding a binding. Please specify a condition.:  2

Updated IAM policy for project [mlops-437709].
bindings:
- members:
  - serviceAccount:service-362026176730@gcp-sa-aiplatform-cc.iam.gserviceaccount.com
  role: roles/aiplatform.customCodeServiceAgent
- members:
  - serviceAccount:service-362026176730@gcp-sa-vertex-op.iam.gserviceaccount.com
  role: roles/aiplatform.onlinePredictionServiceAgent
- members:
  - serviceAccount:service-362026176730@gcp-sa-aiplatform.iam.gserviceaccount.com
  role: roles/aiplatform.serviceAgent
- members:
  - serviceAccount:service-362026176730@gcp-sa-artifactregistry.iam.gserviceaccount.com
  role: roles/artifactregistry.serviceAgent
- members:
  - serviceAccount:feast-sa@mlops-437709.iam.gserviceaccount.com
  role: roles/bigquery.admin
- members:
  - serviceAccount:362026176730@cloudbuild.gserviceaccount.com
  role: roles/cloudbuild.builds.builder
- members:
  - serviceAccount:service-362026176730@gcp-sa-cloudbuild.iam.gserviceaccount.com
  role: roles/cloudbuild.serviceAgent
- members:
  - serviceAccount:service-362026176730@containerregistry.iam.gserviceaccount.com
  role: roles/containerregistry.ServiceAgent
- members:
  - serviceAccount:362026176730-compute@developer.gserviceaccount.com
  role: roles/editor
- members:
  - serviceAccount:service-362026176730@gcp-sa-firestore.iam.gserviceaccount.com
  role: roles/firestore.serviceAgent
- members:
  - serviceAccount:362026176730@cloudbuild.gserviceaccount.com
  role: roles/iam.serviceAccountUser
- members:
  - serviceAccount:service-362026176730@cloud-ml.google.com.iam.gserviceaccount.com
  role: roles/ml.serviceAgent
- members:
  - user:edison@kcl10.com
  role: roles/owner
- members:
  - serviceAccount:service-362026176730@gcp-sa-pubsub.iam.gserviceaccount.com
  role: roles/pubsub.serviceAgent
- members:
  - serviceAccount:362026176730@cloudbuild.gserviceaccount.com
  role: roles/run.admin
- members:
  - serviceAccount:service-362026176730@serverless-robot-prod.iam.gserviceaccount.com
  role: roles/run.serviceAgent
- condition:
    expression: request.time < timestamp("2025-04-09T07:42:05.596Z")
    title: cloudbuild-connection-setup
  members:
  - serviceAccount:service-362026176730@gcp-sa-cloudbuild.iam.gserviceaccount.com
  role: roles/secretmanager.admin
etag: BwY0his6v7Y=
version: 3

建立Key

gcloud iam service-accounts keys create feast-gcp-key.json \
    --iam-account=feast-sa@mlops-437709.iam.gserviceaccount.com

created key [a2609fffff05f5fdf311de233f1a2e1e89288ab5] of type [json] as [feast-gcp-key.json] for [feast-sa@mlops-437709.iam.gserviceaccount.com]

Create K8S Secret

cat feast-gcp-key.json | base64

apiVersion: v1
kind: Secret
metadata:
  name: feast-gcp-key
  namespace: feast
type: Opaque
data:
  key.json: <base64 encoded string>

Online Feature Server

online-feature-server.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: online-feature-server
  namespace: feast
spec:
  replicas: 1
  selector:
    matchLabels:
      app: online-feature-server
  template:
    metadata:
      labels:
        app: online-feature-server
    spec:
      containers:
        - name: online-feature-server
          image: feast:v0.1.8
          ports:
            - containerPort: 8080
          env:
            - name: FEAST_FEATURE_SERVER_CONFIG_PATH
              value: /app/config/feature_store.yaml
            - name: GOOGLE_APPLICATION_CREDENTIALS
              value: /var/secrets/google/key.json
          volumeMounts:
            - name: gcp-sa-key
              mountPath: /var/secrets/google
              readOnly: true
      volumes:
        - name: gcp-sa-key
          secret:
            secretName: gcp-sa-key
---
apiVersion: v1
kind: Service
metadata:
  name: online-feature-server
  namespace: feast
spec:
  selector:
    app: online-feature-server
  type: NodePort
  ports:
    - port: 8080
      targetPort: 8080
      nodePort: 30000

Deploy using Helm

bash install.sh

Step 0: Create feast Namespace
namespace/feast created

Step 1: Deploy Online Store
deployment.apps/online-store created
service/online-store created
Waiting for deployment "online-store" rollout to finish: 0 of 1 updated replicas are available...
deployment "online-store" successfully rolled out

Step 2: Deploy Registry
deployment.apps/registry created
service/registry created
Waiting for deployment "registry" rollout to finish: 0 of 1 updated replicas are available...
deployment "registry" successfully rolled out

Step 3: Deploy Feast Online Feature Server
secret/gcp-sa-key created
deployment.apps/online-feature-server created
service/online-feature-server created
Waiting for deployment "online-feature-server" rollout to finish: 0 of 1 updated replicas are available...
deployment "online-feature-server" successfully rolled out

Test

minikube service online-feature-server -n feast

|-----------|-----------------------|-------------|---------------------------|
| NAMESPACE |         NAME          | TARGET PORT |            URL            |
|-----------|-----------------------|-------------|---------------------------|
| feast     | online-feature-server |        8080 | http://192.168.49.2:30000 |
|-----------|-----------------------|-------------|---------------------------|
🏃  Starting tunnel for service online-feature-server.
|-----------|-----------------------|-------------|------------------------|
| NAMESPACE |         NAME          | TARGET PORT |          URL           |
|-----------|-----------------------|-------------|------------------------|
| feast     | online-feature-server |             | http://127.0.0.1:52316 |
|-----------|-----------------------|-------------|------------------------|
🎉  Opening service feast/online-feature-server in default browser...
❗  Because you are using a Docker driver on darwin, the terminal needs to be open to run it.

curl -X POST http://127.0.0.1:52316/get-online-features \
     -H "Content-Type: application/json" \
     -d @request-get-online-features.json | jq

  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--   100  1010  100   682  100   328  10006   4812 --:--:-- --:--:-- --:--:-- 14852
{
  "metadata": {
    "feature_names": [
      "user_id",
      "transaction_count_7d",
      "credit_score",
      "account_age_days",
      "user_has_2fa_installed",
      "user_has_fraudulent_transactions_7d"
    ]
  },
  "results": [
    {
      "values": [
        "v5zlw0"
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "1970-01-01T00:00:00Z"
      ]
    },
    {
      "values": [
        null
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "1970-01-01T00:00:00Z"
      ]
    },
    {
      "values": [
        480
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "2025-04-29T22:00:34Z"
      ]
    },
    {
      "values": [
        655
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "2025-04-29T22:00:34Z"
      ]
    },
    {
      "values": [
        1
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "2025-04-29T22:00:34Z"
      ]
    },
    {
      "values": [
        0.0
      ],
      "statuses": [
        "PRESENT"
      ],
      "event_timestamps": [
        "2025-05-05T22:00:50Z"
      ]
    }
  ]
}