Go gRPC Advanced-gRPC Conversion HTTP

Preface

We usually use RPC for internal communication and Restful Api for external communication.To avoid writing two sets of apps, we use grpc-gateway Convert gRPC to HTTP.When the service receives an HTTP request, the grpc-gateway converts it to gRPC for processing and then returns the data as JSON.This code builds on the previous one and eventually converts to Restful Api, which supports bearer token validation, data validation, and adding a swagger document.

gRPC to HTTP

Write and compile proto

1. Write simple.proto

syntax = "proto3";

package proto;

import "github.com/mwitkow/go-proto-validators/validator.proto";
import "go-grpc-example/10-grpc-gateway/proto/google/api/annotations.proto";

message InnerMessage {
  // some_integer can only be in range (1, 100).
  int32 some_integer = 1 [(validator.field) = {int_gt: 0, int_lt: 100}];
  // some_float can only be in range (0;1).
  double some_float = 2 [(validator.field) = {float_gte: 0, float_lte: 1}];
}

message OuterMessage {
  // important_string must be a lowercase alpha-numeric of 5 to 30 characters (RE2 syntax).
  string important_string = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}];
  // proto3 doesn't have `required`, the `msg_exist` enforces presence of InnerMessage.
  InnerMessage inner = 2 [(validator.field) = {msg_exists : true}];
}

service Simple{
  rpc Route (InnerMessage) returns (OuterMessage){
      option (google.api.http) ={
          post:"/v1/example/route"
          body:"*"
      };
  }
}

As you can see, proto does not change much, just the route path to the API is added

      option (google.api.http) ={
          post:"/v1/example/route"
          body:"*"
      };

2. Compile simple.proto

The simple.proto file references google/api/annotations.proto( source ) to compile it first.Here I copy the google/folder directly to the proto/directory in the project to compile.If you find that annotations.proto references google/api/http.proto, you compiled it as well.

Go to the directory where annotations.proto is located and compile:

protoc --go_out=plugins=grpc:./ ./http.proto
protoc --go_out=plugins=grpc:./ ./annotations.proto

Go to the directory where simple.proto is located and compile:

#Generate simple.validator.pb.go and simple.pb.go
protoc --govalidators_out=. --go_out=plugins=grpc:./ ./simple.proto
#Generate simple.pb.gw.go
protoc --grpc-gateway_out=logtostderr=true:./ ./simple.proto

This completes the proto compilation and then modifies the service-side code.

Service-side Code Modification

1.Create a new gateway/directory under the server/folder and create a new gateway.go file inside

package gateway

import (
	"context"
	"crypto/tls"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	pb "go-grpc-example/10-grpc-gateway/proto"
	"go-grpc-example/10-grpc-gateway/server/swagger"

	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"golang.org/x/net/http2"
	"golang.org/x/net/http2/h2c"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/grpclog"
)

// ProvideHTTP transforms gRPC services into HTTP services, enabling gRPC to support HTTP at the same time
func ProvideHTTP(endpoint string, grpcServer *grpc.Server) *http.Server {
	ctx := context.Background()
	//Get Certificates
	creds, err := credentials.NewClientTLSFromFile("../tls/server.pem", "go-grpc-example")
	if err != nil {
		log.Fatalf("Failed to create TLS credentials %v", err)
	}
	//Add Certificate
	dopts := []grpc.DialOption{grpc.WithTransportCredentials(creds)}
	//Create a new gwmux, which is the request multiplexer for grpc-gateway.It matches the http request to the pattern and calls the corresponding handler.
	gwmux := runtime.NewServeMux()
	//Register the service's http handler with gwmux.Handler forwards request to grpc endpoint via endpoint
	err = pb.RegisterSimpleHandlerFromEndpoint(ctx, gwmux, endpoint, dopts)
	if err != nil {
		log.Fatalf("Register Endpoint err: %v", err)
	}
	//New mux, which is the request multiplexer for http
	mux := http.NewServeMux()
	//Register gwmux
	mux.Handle("/", gwmux)
	log.Println(endpoint + " HTTP.Listing whth TLS and token...")
	return &http.Server{
		Addr:      endpoint,
		Handler:   grpcHandlerFunc(grpcServer, mux),
		TLSConfig: getTLSConfig(),
	}
}

// grpcHandlerFunc redirects to the specified Handler processing based on different requests
func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler {
	return h2c.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") {
			grpcServer.ServeHTTP(w, r)
		} else {
			otherHandler.ServeHTTP(w, r)
		}
	}), &http2.Server{})
}

// getTLSConfig Gets TLS Configuration
func getTLSConfig() *tls.Config {
	cert, _ := ioutil.ReadFile("../tls/server.pem")
	key, _ := ioutil.ReadFile("../tls/server.key")
	var demoKeyPair *tls.Certificate
	pair, err := tls.X509KeyPair(cert, key)
	if err != nil {
		grpclog.Fatalf("TLS KeyPair err: %v\n", err)
	}
	demoKeyPair = &pair
	return &tls.Config{
		Certificates: []tls.Certificate{*demoKeyPair},
		NextProtos:   []string{http2.NextProtoTLS}, // HTTP2 TLS support
	}
}

Its main purpose is to redirect unused requests to the specified service for processing, thereby enabling the transfer of HTTP requests to the gRPC service.

2.gRPC supports HTTP

    //Convert grpcServer to httpServer using gateway
	httpServer := gateway.ProvideHTTP(Address, grpcServer)
	if err = httpServer.Serve(tls.NewListener(listener, httpServer.TLSConfig)); err != nil {
		log.Fatal("ListenAndServe: ", err)
	}

Using the postman test

As you can see in the diagram, our gRPC service already supports both RPC and HTTP requests, and the API interface supports bearer token and data validation.To facilitate docking, we generate a swagger document from the API interface.

Generate swagger document

Generate swagger document - simple.swagger.json

1. Install protoc-gen-swagger

go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger

2. Compile to generate simple.swagger.json

To the simple.proto file directory, compile:
protoc --swagger_out=logtostderr=true:./ ./simple.proto

Again, I use the VSCode-proto3 plug-in in VSCode. First It is introduced that as long as it is saved, it will be automatically compiled, which is very convenient and does not need to memorize instructions.The complete configuration is as follows:

    // vscode-proto3 Plugin Configuration
    "protoc": {
        // Directory where protoc.exe is located
        "path": "C:\\Go\\bin\\protoc.exe",
        // Compile automatically on save
        "compile_on_save": true,
        "options": [
            "--go_out=plugins=grpc:.",//Compile the output.pb.go file in the current directory
            "--govalidators_out=.",//Compile the output.validator.pb file in the current directory
            "--grpc-gateway_out=logtostderr=true:.",//Compile the output.pb.gw.go file in the current directory
            "--swagger_out=logtostderr=true:."//Compile the output.swagger.json file in the current directory
        ]
    }

After compilation and generation, leave the required files behind and delete them unnecessarily.

Convert swagger-ui to Go code, alternate

1. Download swagger-ui

Download Address To copy all the files in the dist directory to the server/swagger/swagger-ui/directory of our project.

2. Convert Swagger UI to Go Code

Install go-bindata:
go get -u github.com/jteeuwen/go-bindata/...

Go back to the server/directory and run the command to turn the Swagger UI into Go code.
go-bindata --nocompress -pkg swagger -o swagger/datafile.go swagger/swagger-ui/...

  • This is a tricky step, and you have to go back to the directory where the main function is located to run the instructions, because the _bindata in the generated Go code maps the paths of the swagger-ui from which the program looks for pages.If you do not run the command in the directory where the main function is located, the resulting path is incorrect, and 404 will be reported and the page cannot be found.The main function on the server/side of this project is in server.go, so run the command in the server/directory.
var _bindata = map[string]func() (*asset, error){
	"swagger/swagger-ui/favicon-16x16.png": swaggerSwaggerUiFavicon16x16Png,
	"swagger/swagger-ui/favicon-32x32.png": swaggerSwaggerUiFavicon32x32Png,
	"swagger/swagger-ui/index.html": swaggerSwaggerUiIndexHtml,
	"swagger/swagger-ui/oauth2-redirect.html": swaggerSwaggerUiOauth2RedirectHtml,
	"swagger/swagger-ui/swagger-ui-bundle.js": swaggerSwaggerUiSwaggerUiBundleJs,
	"swagger/swagger-ui/swagger-ui-bundle.js.map": swaggerSwaggerUiSwaggerUiBundleJsMap,
	"swagger/swagger-ui/swagger-ui-standalone-preset.js": swaggerSwaggerUiSwaggerUiStandalonePresetJs,
	"swagger/swagger-ui/swagger-ui-standalone-preset.js.map": swaggerSwaggerUiSwaggerUiStandalonePresetJsMap,
	"swagger/swagger-ui/swagger-ui.css": swaggerSwaggerUiSwaggerUiCss,
	"swagger/swagger-ui/swagger-ui.css.map": swaggerSwaggerUiSwaggerUiCssMap,
	"swagger/swagger-ui/swagger-ui.js": swaggerSwaggerUiSwaggerUiJs,
	"swagger/swagger-ui/swagger-ui.js.map": swaggerSwaggerUiSwaggerUiJsMap,
}

Offering swagger-ui to the outside world

1. Create a new swagger.go file in the swagger/directory

package swagger

import (
	"log"
	"net/http"
	"path"
	"strings"

	assetfs "github.com/elazarl/go-bindata-assetfs"
)

//ServeSwaggerFile Exposes the swagger.json file in the proto folder
func ServeSwaggerFile(w http.ResponseWriter, r *http.Request) {
	if !strings.HasSuffix(r.URL.Path, "swagger.json") {
		log.Printf("Not Found: %s", r.URL.Path)
		http.NotFound(w, r)
		return
	}

	p := strings.TrimPrefix(r.URL.Path, "/swagger/")
	// ". /proto/" is the directory of.swagger.json
	p = path.Join("../proto/", p)

	log.Printf("Serving swagger-file: %s", p)

	http.ServeFile(w, r, p)
}

//ServeSwaggerUI provides swagger-ui
func ServeSwaggerUI(mux *http.ServeMux) {
	fileServer := http.FileServer(&assetfs.AssetFS{
		Asset:    Asset,
		AssetDir: AssetDir,
		Prefix:   "swagger/swagger-ui", //The directory where the swagger-ui folder is located
	})
	prefix := "/swagger-ui/"
	mux.Handle(prefix, http.StripPrefix(prefix, fileServer))
}

2. Register swagger

Add the following code to gateway.go

	//Register swagger
	mux.HandleFunc("/swagger/", swagger.ServeSwaggerFile)
	swagger.ServeSwaggerUI(mux)

We've finished adding swagger documents here. Since Google Browser can't use its own TLS certificate, we're testing with Firefox Browser.

Open with Firefox Browser: https://127.0.0.1:8000/swagger-ui/

Enter in the top address bar: https://127.0.0.1:8000/swagger/simple.swagger.json

Then you can see the API documentation generated by swagger.

Another question is, how do we add bearer token to swagger when we use bearer token for interface validation?
Last I did this on grpc-gatewayGitHub Issues Find a solution.

Configuring bearer token in swagger

1. Modify the simple.proto file

syntax = "proto3";

package proto;

import "github.com/mwitkow/go-proto-validators/validator.proto";
import "go-grpc-example/10-grpc-gateway/proto/google/api/annotations.proto";
import "go-grpc-example/10-grpc-gateway/proto/google/options/annotations.proto";

message InnerMessage {
  // some_integer can only be in range (1, 100).
  int32 some_integer = 1 [(validator.field) = {int_gt: 0, int_lt: 100}];
  // some_float can only be in range (0;1).
  double some_float = 2 [(validator.field) = {float_gte: 0, float_lte: 1}];
}

message OuterMessage {
  // important_string must be a lowercase alpha-numeric of 5 to 30 characters (RE2 syntax).
  string important_string = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}];
  // proto3 doesn't have `required`, the `msg_exist` enforces presence of InnerMessage.
  InnerMessage inner = 2 [(validator.field) = {msg_exists : true}];
}

option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {
  security_definitions: {
    security: {
      key: "bearer"
      value: {
        type: TYPE_API_KEY
        in: IN_HEADER
        name: "Authorization"
        description: "Authentication token, prefixed by Bearer: Bearer <token>"
      }
    }
  }

  security: {
    security_requirement: {
      key: "bearer"
    }
  }

  info: {
		title: "grpc gateway sample";
		version: "1.0";	
		license: {
			name: "MIT";			
		};
  }

  schemes: HTTPS
};

service Simple{
  rpc Route (InnerMessage) returns (OuterMessage){
      option (google.api.http) ={
          post:"/v1/example/route"
          body:"*"
      };
      // //Disable bearer token
      // option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = {
      //   security: { } // Disable security key
      // };
  }
}

2. Recompile to generate simple.swagger.json

Be accomplished!

Validation Tests

1. Add bearer token

2. Call the interface and return the data correctly

3. Transfer irregular data and return data validation logic errors

summary

This article describes how to use grpc-gateway to enable gRPC to support HTTP at the same time, and ultimately convert to Restful Api to support bearer token validation, data validation.A swagger document is also generated to facilitate docking of API interfaces.

Tutorial source address: https://github.com/Bingjian-Zhu/go-grpc-example

Reference documents:
https://eddycjy.com/tags/grpc-gateway/
https://segmentfault.com/a/1190000008106582

Tags: Google JSON github Firefox

Posted on Tue, 05 May 2020 20:47:11 -0400 by mrwhale