Create a simple API server image for docker and Kubernetes

Summary

We will be creating our own backend API server that we can use to test out our Kubernetes deployment. We will also be running this on a docker-compose setup for testing as well. We will connect to Mongo Database, Redis, and Influx Database to store and query our data.

API endpoint that we will create:

path "/", is just a GET method to test if we can reach our API-server, it will just return a simple JSON object {"hello": "world"}

path "/note", a POST method for users to insert their notes into the database, and the GET method to retrieve their desired notes.

Prerequisites

Some things to installed beforehand:

Golang
Docker engine and Docker Compose

Writing API server code

initialize go mod file and install necessary dependencies

 mkdir api-server
 cd api-server
 go mod init api-server

 # install dependencies
 go get github.com/gin-gonic/gin
 go get go.mongodb.org/mongo-driver/mongo
 go get github.com/go-redis/redis/v9
 go get github.com/influxdata/influxdb-client-go/v2

write a simple gin router to start a server in main.go

 // main.go

 package main
 import (
     ...
 )

 func main() {
     fmt.Println("homek8")
     }()

     r := gin.Default()

     // simple get function to test
     r.GET("/", func(c *gin.Context) {
         c.JSON(http.StatusOK, gin.H{
             "hello": "world",
         })
     })

     r.Run()
 }

write the clients for Mongo, Redis, and Influxdb to connect to the database.

Once done we can add the connection to our main function. Remember to close the connection for the clients by adding it to the defer statement.

 // main.go

 var influxToken = os.Getenv("DOCKER_INFLUXDB_INIT_ADMIN_TOKEN")

 var ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
 var redisCtx = context.Background()
 var influxCtx = context.Background()

 func MongoClientInit() *mongo.Client {
     defer cancel()
     address := "mongodb://localhost:27017"
     mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(address))
     if err != nil {
         panic(err)
     }
     if err := mongoClient.Ping(ctx, nil); err != nil {
         panic(err)
     }

     fmt.Println("Connect to mongodb OK")
     return mongoClient
 }

 func RedisClientInit() *redis.Client {
     address := "localhost:6379"
     rdb := redis.NewClient(&redis.Options{
         Addr:     address,
         Password: "",
         DB:       0,
     })

     if err := rdb.Ping(redisCtx); err.Err() != nil {
         panic(err)
     }

     fmt.Println("Connect to redis OK")
     return rdb
 }

 func InfluxClientInit() *influxdb2.Client {
     address := "http://localhost:8086"
     ifxdb := influxdb2.NewClient(address, influxToken)
     ok, err := ifxdb.Ping(influxCtx)
     if err != nil {
         panic(err)
     }
     if !ok {
         panic("Unable to connect to influxdb")
     }
     fmt.Println("Connect to influx OK")
     return &ifxdb
 }

 func main() {
     fmt.Println("homek8")
     fmt.Println("connecting to clients ...")
     redisClient := RedisClientInit()
     mongoClient := MongoClientInit()
     influxClient := InfluxClientInit()
     fmt.Println("connected")

     defer func() {
         if err := mongoClient.Disconnect(ctx); err != nil {
             panic(err)
         }
         (*influxClient).Close()
     }()

     ...
 }

write our API handler for /note

The POST handler /note will write the title and value information into our Mongo database.

The GET handler /note will first try to retrieve the data from the Redis database. If that fails, it will query the Mongo database and update the entry into the Redis database.

 // main.go

 type Note struct {
     Title string `json:"title"`
     Value string `json:"value"`
 }

 func main() {
     ...initiallization...
     coll := mongoClient.Database("homek8").Collection("notes")

     r.GET("/note", func(c *gin.Context) {
         name, _ := c.Params.Get("name")

         val, err := redisClient.Get(redisCtx, name).Result()
         if err != nil && err != redis.Nil {
             log.Println(err)
             c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get note"})
             return
         }
         if err == redis.Nil {
             fmt.Println("found redis value:", val)
             c.JSON(http.StatusOK, Note{Title: name, Value: val})
             return
         }

         var note Note
         filter := bson.D{{Key: "name", Value: name}}
         err = coll.FindOne(ctx, filter).Decode(&note)
         if err != nil {
             c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get note"})
             return
         }

         err = redisClient.Set(redisCtx, name, note.Value, 0).Err()
         if err != nil {
             fmt.Println("failed to cache data into redis")
         }
         c.JSON(http.StatusOK, note)
     })

     r.POST("/note", func(c *gin.Context) {
         var note Note
         err := c.BindJSON(&note)
         if err != nil {
             fmt.Println("unmarshall: ", err)
             c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to add a note"})
             return
         }

         res, err := coll.InsertOne(ctx, note)
         if err != nil {
             fmt.Println("mongo error: ", err)
             c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to add a note"})
             return
         }
         id := res.InsertedID
         c.JSON(http.StatusOK, gin.H{"success": id})
     })

     ...
 }

write a go routine to send data to influxdb

we will send random average and max values to our influxdb database. It will emit one value once every 5 seconds for 100 times

 func main() {
     writeAPI := (*influxClient).WriteAPIBlocking("andre", "bucket1")
     go func() {
         count := 100
         avg := 24.5
         max := 45.0
         for count > 0 {
             avgDeviation := rand.Intn(20) + 1
             maxDeviation := rand.Intn(6) + 0
             pt := influxdb2.NewPoint("stat",
                 map[string]string{"unit": "temperature"},
                 map[string]interface{}{"avg": avg + float64(avgDeviation), "max": max + float64(maxDeviation)},
                 time.Now())
             err := writeAPI.WritePoint(influxCtx, pt)
             if err != nil {
                 fmt.Println("emit point error", err)
             } else {
                 fmt.Println("success sent to influx db")
             }
             time.Sleep(time.Second * 5)
             count -= 1
         }
     }()
 }

run go build to check if there are no errors

Create a Dockerfile

Let's create a docker file for us to containerize our application by creating a Dockerfile

# image will be build using Go version 1.21
FROM golang:1.21

# sets the current working directory to /app
WORKDIR /app
# copy our go.mod and go.sum into our current working directory
COPY go.mod go.sum ./
# install our go dependencies
RUN go mod download
# copy our main.go (or all go files) to our current working directory
COPY *.go ./
# build our application and name our binary file as output
RUN go build -o output

# expose 8080 in our container
EXPOSE 8080

# execut command to launch our appliaction
CMD [ "/app/output" ]

Here, we can manually build our image using this Dockerfile by running: docker build . .It will search for any Dockerfile in the current directory and build the image.

We can also tag our images when building with a -t <tag>

To view all created images, run docker images

Create a docker-compose file

Now we can write a docker-composee.yml file to spin up the containers we need. We need 4 containers, our backend server we had just written, a mongo db container, a redis db container, and an influx db container. We can easily spin up multiple docker containers with a single command with docker-compose.

services:
  backend:
    build: .
    ports:
      - "8080:8080"
    environment:
      - ENV=dockercompose
      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-token
    depends_on:
      - mongo
      - redis
      - influx
  mongo:
    image: mongo:6.0
    ports:
      - "27017:27017"
  redis:
    image: redis:alpine
    ports:
      - "6379:6379"
  influx:
    image: influxdb:alpine
    environment:
      - DOCKER_INFLUXDB_INIT_USERNAME=andre
      - DOCKER_INFLUXDB_INIT_PASSWORD=12345678
      - DOCKER_INFLUXDB_INIT_MODE=setup
      - DOCKER_INFLUXDB_INIT_ORG=andre
      - DOCKER_INFLUXDB_INIT_BUCKET=bucket1
      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-token
    ports:
      - "8086:8086"

In our docker-compose.yml file, we list out the names of the services that we need: backend, mongo, redis, influx.

Under the backend, we have build . which will allow Docker Compose to look for the Dockerfile we had written earlier and use that to build the image for this service. For the other services, we will be using images from Docker Hub, for example, mongo:6.0 , redis:alpine.

The ports we defined above also expose the ports of each container to our host.

We also added a new environment variable ENV=dockercompose to our backend container. This allows us to change the URI of our databases easily. We also pass in our influx admin token to the backend so it can successfully emit data to the influx database.

depends_on over here will wait for Mongo, redis, and influx services to be up and running first before running the backend service.

Now we need to update our backend code to include this ENV variable. We can get the ENV environment variable through the code and use it to set the client address URL we will connect to. We replace localhost here to the name of our service which will then automatically resolve to the correct IP address.

Under the hood, docker-compose sets up its own DNS resolution that maps the service name to its IP address. This is used for communications between containers.

// main.go

var ENV = os.Getenv("ENV")

func MongoClientInit() *mongo.Client {
    defer cancel()
    address := "mongodb://localhost:27017"
    if ENV == "dockercompose" {
        address = "mongodb://mongo:27017"
    }
    mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(address))
    if err != nil {
        panic(err)
    }
    if err := mongoClient.Ping(ctx, nil); err != nil {
        panic(err)
    }

    fmt.Println("Connect to mongodb OK")
    return mongoClient
}

func RedisClientInit() *redis.Client {
    address := "localhost:6379"
    if ENV == "dockercompose" {
        address = "redis:6379"
    }
    rdb := redis.NewClient(&redis.Options{
        Addr:     address,
        Password: "",
        DB:       0,
    })

    if err := rdb.Ping(redisCtx); err.Err() != nil {
        panic(err)
    }

    fmt.Println("Connect to redis OK")
    return rdb
}

func InfluxClientInit() *influxdb2.Client {
    address := "http://localhost:8086"
    if ENV == "dockercompose" {
        address = "http://influx:8086"
    }
    ifxdb := influxdb2.NewClient(address, influxToken)
    ok, err := ifxdb.Ping(influxCtx)
    if err != nil {
        panic(err)
    }
    if !ok {
        panic("Unable to connect to influxdb")
    }
    fmt.Println("Connect to influx OK")
    return &ifxdb
}

Verify if it's working

Now we can run containers using docker-compose: docker compose up. We should be able to see 4 containers running.

To stop running our containers : docker compose down

Docker Image

Now we can push our image to our image repository. Here we will be pushing to Docker hub. After creating an account on the website, we can create a new repository as shown below.

We can then build our image with the command: docker build . -t andrewongzh/api-server:1.0.0

we might need to login to docker hub first in the terminal

run docker login -u <username>

After that, we can push our image with the command: docker push andrewongzh/api-server:1.0.0. This is in the format of <username>/<image name>:<image tag>

one thing to note here is to tag the image name correctly to match the format of how we are pushing to the repository

If everything works well, we can see our image on Docker hub.

In conclusion, we created a simple backend server to play around with docker and docker-compose. We will use this server for more things in the future.

Handle secrets

Some extra we can do is to move our secrets into a file instead of writing it on the docker-compose.yml file.

First, we can create a folder to store our secrets and then create influxtoken.txt and passwd.txt under it.

mkdir secrets
echo my-token > ./secrets/influxtoken.txt
echo 12345678 > ./secrets/passwd.txt

In the docker-compose.yml file make the following changes:

services:
  backend:
    build: .
    ports:
      - "8080:8080"
    environment:
      - ENV=dockercompose
      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=/run/secrets/influxtoken
    depends_on:
      - mongo
      - redis
      - influx
    secrets:
      - passwd
  mongo:
    image: mongo:6.0
    ports:
      - "27017:27017"
  redis:
    image: redis:alpine
    ports:
      - "6379:6379"
  influx:
    image: influxdb:alpine
    environment:
      - DOCKER_INFLUXDB_INIT_USERNAME=andre
      - DOCKER_INFLUXDB_INIT_PASSWORD=/run/secrets/passwd
      - DOCKER_INFLUXDB_INIT_MODE=setup
      - DOCKER_INFLUXDB_INIT_ORG=andre
      - DOCKER_INFLUXDB_INIT_BUCKET=bucket1
      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=/run/secrets/influxtoken
    ports:
      - "8086:8086"
secrets:
  passwd:
    file: ./secrets/passwd.txt
  influxtoken:
    file: ./secrets/influxtoken.txt

Appendix

// full code of main.go

package main

import (
    "context"
    "fmt"
    "log"
    "math/rand"
    "net/http"
    "os"
    "time"

    "github.com/gin-gonic/gin"
    influxdb2 "github.com/influxdata/influxdb-client-go/v2"
    "github.com/redis/go-redis/v9"
    "go.mongodb.org/mongo-driver/bson"
    "go.mongodb.org/mongo-driver/mongo"
    "go.mongodb.org/mongo-driver/mongo/options"
)

type Note struct {
    Title string `json:"title"`
    Value string `json:"value"`
}

var ENV = os.Getenv("ENV")
var influxToken = os.Getenv("DOCKER_INFLUXDB_INIT_ADMIN_TOKEN")

var ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
var redisCtx = context.Background()
var influxCtx = context.Background()

func MongoClientInit() *mongo.Client {
    defer cancel()
    address := "mongodb://localhost:27017"
    if ENV == "dockercompose" {
        address = "mongodb://mongo:27017"
    }
    mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(address))
    if err != nil {
        panic(err)
    }
    if err := mongoClient.Ping(ctx, nil); err != nil {
        panic(err)
    }

    fmt.Println("Connect to mongodb OK")
    return mongoClient
}

func RedisClientInit() *redis.Client {
    address := "localhost:6379"
    if ENV == "dockercompose" {
        address = "redis:6379"
    }
    rdb := redis.NewClient(&redis.Options{
        Addr:     address,
        Password: "",
        DB:       0,
    })

    if err := rdb.Ping(redisCtx); err.Err() != nil {
        panic(err)
    }

    fmt.Println("Connect to redis OK")
    return rdb
}

func InfluxClientInit() *influxdb2.Client {
    address := "http://localhost:8086"
    if ENV == "dockercompose" {
        address = "http://influx:8086"
    }
    ifxdb := influxdb2.NewClient(address, influxToken)
    ok, err := ifxdb.Ping(influxCtx)
    if err != nil {
        panic(err)
    }
    if !ok {
        panic("Unable to connect to influxdb")
    }
    fmt.Println("Connect to influx OK")
    return &ifxdb
}

func main() {
    fmt.Println("homek8")
    fmt.Println("connecting to clients ...")
    redisClient := RedisClientInit()
    mongoClient := MongoClientInit()
    influxClient := InfluxClientInit()
    fmt.Println("connected")

    defer func() {
        if err := mongoClient.Disconnect(ctx); err != nil {
            panic(err)
        }
        (*influxClient).Close()
    }()
    coll := mongoClient.Database("homek8").Collection("notes")
    writeAPI := (*influxClient).WriteAPIBlocking("andre", "bucket1")

    // we shall emit data every 5 seconds for 100 times
    go func() {
        count := 100
        avg := 24.5
        max := 45.0
        for count > 0 {
            avgDeviation := rand.Intn(20) + 1
            maxDeviation := rand.Intn(6) + 0
            pt := influxdb2.NewPoint("stat",
                map[string]string{"unit": "temperature"},
                map[string]interface{}{"avg": avg + float64(avgDeviation), "max": max + float64(maxDeviation)},
                time.Now())
            err := writeAPI.WritePoint(influxCtx, pt)
            if err != nil {
                fmt.Println("emit point error", err)
            } else {
                fmt.Println("success sent to influx db")
            }
            time.Sleep(time.Second * 5)
            count -= 1
        }
    }()

    r := gin.Default()

    // simple get function to test
    r.GET("/", func(c *gin.Context) {
        c.JSON(http.StatusOK, gin.H{
            "hello": "world",
        })
    })

    r.GET("/note", func(c *gin.Context) {
        name, _ := c.Params.Get("name")

        val, err := redisClient.Get(redisCtx, name).Result()
        if err != nil && err != redis.Nil {
            log.Println(err)
            c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get note"})
            return
        }
        if err == redis.Nil {
            fmt.Println("found redis value:", val)
            c.JSON(http.StatusOK, Note{Title: name, Value: val})
            return
        }

        var note Note
        filter := bson.D{{Key: "name", Value: name}}
        err = coll.FindOne(ctx, filter).Decode(&note)
        if err != nil {
            c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get note"})
            return
        }

        err = redisClient.Set(redisCtx, name, note.Value, 0).Err()
        if err != nil {
            fmt.Println("failed to cache data into redis")
        }
        c.JSON(http.StatusOK, note)
    })

    r.POST("/note", func(c *gin.Context) {
        var note Note
        err := c.BindJSON(&note)
        if err != nil {
            fmt.Println("unmarshall: ", err)
            c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to add a note"})
            return
        }

        res, err := coll.InsertOne(ctx, note)
        if err != nil {
            fmt.Println("mongo error: ", err)
            c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to add a note"})
            return
        }
        id := res.InsertedID
        c.JSON(http.StatusOK, gin.H{"success": id})
    })

    r.Run()
}

Create a simple API server image for docker and Kubernetes

Comments

More from this blog

Openclaw introduction

Running ollama on Proxmox VM

2024 Home lab setup

Creating my own storage server

Using Gitea as a private image repository

Summary

Prerequisites

Writing API server code

Create a Dockerfile

Create a docker-compose file

Verify if it's working

Docker Image

Handle secrets

Appendix

Command Palette

Comments

More from this blog

Summary

Prerequisites

Writing API server code

Create a Dockerfile

Create a docker-compose file

Verify if it's working

Docker Image

Handle secrets

Appendix