Deploying to K3s

- 3 minutes read - 578 words

A simple deployment of cAdvisor to K3s and to confirm the ability to expose Ingresses using Tailscale Kubernetes Operator (TLS) and – since it’s already installed with K3s – Traefik (non-TLS).

local image = "gcr.io/cadvisor/cadvisor:latest";

local labels = {
  app: "cadvisor",
};

local name = std.extVar("NAME");

local node_ip = std.extVar("NODE_IP");

local port = 8080;

local deployment = {
  apiVersion: "apps/v1",
  kind: "Deployment",
  metadata: {
    name: name,
    labels: labels,
  },
  spec: {
    replicas: 1,
    selector: {
      matchLabels: labels,
    },
    template: {
      metadata: {
        labels: labels,
      },
      spec: {
        containers: [
          {
            name: name,
            image: image,
            ports: [
              {
                name: "http",
                containerPort: 8080,
                protocol: "TCP",
              },
            ],
            resources: {
              limits: {
                memory: "500Mi",
              },
              requests: {
                cpu: "250m",
                memory: "250Mi",
              },
            },
            securityContext: {
              allowPrivilegeEscalation: false,
              privileged: false,
              readOnlyRootFilesystem: true,
              runAsGroup: 1000,
              runAsNonRoot: true,
              runAsUser: 1000,
            },
          },
        ],
      },
    },
  },
};


local ingresses = [
  {
    // Tailscale Ingress TLS (non-public)
    apiVersion: "networking.k8s.io/v1",
    kind: "Ingress",
    metadata: {
      name: "tailscale",
      labels: labels,
    },
    spec: {
      ingressClassName: "tailscale",
      defaultBackend: {
        service: {
          name: name,
          port: {
            number: port,
          },
        },
      },
      tls: [
        {
          hosts: [
            name,
          ],
        },
      ],
    },
  },
  {
    // Traefik Ingress non-TLS (non-public)
    apiVersion: "networking.k8s.io/v1",
    kind: "Ingress",
    metadata: {
      name: "traefik",
      labels: labels,
    },
    spec: {
      ingressClassName: "traefik",
      rules: [
        {
          host: std.format(
              "%(name)s.%(node_ip)s.nip.io", {
                name: name,
                node_ip: node_ip,
              },
            ),
          http: {
            paths: [
              {
                path: "/",
                pathType: "Prefix",
                backend: {
                  service: {
                    name: name,
                    port: {
                      number: port,
                    },
                  },
                },
              },
            ],
          },
        },
      ],
    },
  },
];

local prometheusrule = {
  // Alternatively parameterize the duration
  local duration = 10,

  apiVersion: "monitoring.coreos.com/v1",
  kind: "PrometheusRule",
  metadata: {
    name: name,
    labels: labels,
  },
  spec: {
    groups: [
      {
        name: name,
        rules: [
          {
            alert: "TestcAdvsiorDown",
            annotations: {
              summary: "Test cAdvisor instance is down or not scraping metrics.",
              description: std.format(
                "The cAdvisor instance {{ $labels.instance }} has not been scraping metrics for more than %(duration)s minutes.", {
                  duration: duration,
                },
              ),
            },
            expr: "absent(cadvisor_version_info{namespace=\"k3s-test\"}) or (cadvisor_version_info{namespace=\"k3s-test\"}!=1)",
            "for": std.format(
              "%(duration)sm", {
                duration: duration,
              },
            ),
            labels: labels {
              severity: "warning",
            },
          },
        ],
      },
    ],
  },
};

local service = {
  apiVersion: "v1",
  kind: "Service",
  metadata: {
    name: name,
    labels: labels,
  },
  spec: {
    selector: labels,
    ports: [
      {
        name: "http",
        port: port,
        targetPort: port,
        protocol: "TCP",
      },
    ],
  },
};

local serviceaccount = {
  apiVersion: "v1",
  kind: "ServiceAccount",
  metadata: {
    name: name,
    labels: labels,
  },
};

local servicemonitor = {
  apiVersion: "monitoring.coreos.com/v1",
  kind: "ServiceMonitor",
  metadata: {
    name: name,
    labels: labels,
  },
  spec: {
    selector: {
      matchLabels: labels,
    },
    endpoints: [
      {
        interval: "120s",
        path: "/metrics",
        port: "http",
        scrapeTimeout: "30s",
      },
    ],
  },
};

// Output
{
  apiVersion: "v1",
  kind: "List",
  items: [
    deployment,
    prometheusrule,
    service,
    serviceaccount,
    servicemonitor,
  ] + ingresses,
}

I like to have a script that applies Jsonnet to the file:

#!/usr/bin/env bash

NAME="test"
TAILNET="...ts.net"

# Get the internal IP of the first (only) node to be used by Traefik Ingress
NODE_IP=$(\
  kubectl get nodes \
  --output=jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'
)

jsonnet \
--ext-str NAME="${NAME}" \
--ext-str NODE_IP="${NODE_IP}" \
kubernetes.jsonnet

and apply it using:

NAMESPACE="..."

./kubernetes.sh \
| jq -r . \
| kubectl apply \
  --filename=- \
  --namespace=${NAMESPACE}

This should yield:

kubectl get ingress --namespace=${NAMESPACE}

NAME      CLASS       HOSTS                   ADDRESS        PORTS     AGE
test      tailscale   *                       test.{TAILNET} 80, 443   19h
traefik   traefik     test.{NODE_IP}.nip.io   {NODE_IP}      80        46m

And:

NAME="test"
TAILNET="...ts.net"
NODE_IP="..."

ENDPOINTS=(
    "https://${NAME}.${TAILNET}"
    "http://${NAME}.${NODE_IP}.nip.io"
)

for ENDPOINT in "${ENDPOINTS[@]}"
do
curl \
--silent \
--url "${ENDPOINT}/metrics" \
--output /dev/null \
--write-out '%{response_code}\n'
done

200
200

That’s all!