Feature: generating protoflles with docker. (#4427)

* Gen proto files.

* Clean up.

* Fix.

* Buildx multiplatform.

* Added platform.

* Fix generate.

* Fix generate.

* Fix generate.
pull/4436/head
Konstantin 2 years ago committed by GitHub
parent d2a34630ef
commit a18e1e44bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      .travis.yml
  2. 3
      Makefile
  3. 4
      api/proto/message/gen.sh
  4. 2
      api/proto/message/message.go
  5. 82
      api/proto/message/message.pb.go
  6. 12
      api/proto/message/message_grpc.pb.go
  7. 9
      api/service/legacysync/downloader/Proto.Dockerfile
  8. 4
      api/service/legacysync/downloader/build.sh
  9. 5
      api/service/legacysync/downloader/gen.sh
  10. 3
      api/service/legacysync/downloader/proto/downloader.go
  11. 4
      api/service/legacysync/downloader/proto/downloader.pb.go
  12. 12
      api/service/legacysync/downloader/proto/downloader_grpc.pb.go
  13. 9
      api/service/legacysync/downloader/proto/gen.sh
  14. 3
      p2p/stream/protocols/sync/message/gen.sh
  15. 2
      p2p/stream/protocols/sync/message/generate.go
  16. 4
      p2p/stream/protocols/sync/message/msg.pb.go
  17. 2
      scripts/gogenerate.sh
  18. 15
      scripts/install_build_tools.sh
  19. 66
      scripts/install_protoc.sh

@ -22,13 +22,10 @@ install:
- git clone https://github.com/harmony-one/harmony-test.git $GOPATH/src/github.com/harmony-one/harmony-test
- (cd $GOPATH/src/github.com/harmony-one/mcl; make -j4)
- (cd $GOPATH/src/github.com/harmony-one/bls; make BLS_SWAP_G=1 -j4)
- go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
- go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
# - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.1
- make go-get
- go install golang.org/x/tools/cmd/goimports@latest
- go install github.com/harmony-ek/gencodec@latest
- scripts/install_build_tools.sh
script:
- ${TEST}
after_success:

@ -164,3 +164,6 @@ go-test:
docker:
docker build --pull -t harmonyone/$(PKGNAME):latest -f scripts/docker/Dockerfile .
travis_go_checker:
bash ./scripts/travis_go_checker.sh

@ -1 +1,3 @@
protoc -I ./ message.proto --go_out=. --go-grpc_out=.
#!/bin/bash
docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/message.proto

@ -1,3 +1,3 @@
package message
//go:generate protoc message.proto --go_out=. --go-grpc_out=.
//go:generate ./gen.sh

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.15.8
// protoc-gen-go v1.30.0
// protoc v3.12.4
// source: message.proto
package message
@ -25,9 +25,9 @@ type ServiceType int32
const (
ServiceType_CONSENSUS ServiceType = 0
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
ServiceType_STAKING ServiceType = 1
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
ServiceType_DRAND ServiceType = 2
ServiceType_CLIENT_SUPPORT ServiceType = 3
)
@ -79,7 +79,7 @@ func (ServiceType) EnumDescriptor() ([]byte, []int) {
type MessageType int32
const (
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
MessageType_NEWNODE_BEACON_STAKING MessageType = 0
MessageType_ANNOUNCE MessageType = 1
MessageType_PREPARE MessageType = 2
@ -88,11 +88,11 @@ const (
MessageType_COMMITTED MessageType = 5
MessageType_VIEWCHANGE MessageType = 6
MessageType_NEWVIEW MessageType = 7
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
MessageType_DRAND_INIT MessageType = 10
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
MessageType_DRAND_COMMIT MessageType = 11
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
MessageType_LOTTERY_REQUEST MessageType = 12 // it should be either ENTER or GETPLAYERS but it will be removed later.
)
@ -156,11 +156,11 @@ func (MessageType) EnumDescriptor() ([]byte, []int) {
type LotteryRequest_Type int32
const (
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
LotteryRequest_ENTER LotteryRequest_Type = 0
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
LotteryRequest_RESULT LotteryRequest_Type = 1
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
LotteryRequest_PICK_WINNER LotteryRequest_Type = 2
)
@ -288,7 +288,7 @@ func (m *Message) GetRequest() isMessage_Request {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *Message) GetStaking() *StakingRequest {
if x, ok := x.GetRequest().(*Message_Staking); ok {
return x.Staking
@ -303,7 +303,7 @@ func (x *Message) GetConsensus() *ConsensusRequest {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *Message) GetDrand() *DrandRequest {
if x, ok := x.GetRequest().(*Message_Drand); ok {
return x.Drand
@ -318,7 +318,7 @@ func (x *Message) GetViewchange() *ViewChangeRequest {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *Message) GetLotteryRequest() *LotteryRequest {
if x, ok := x.GetRequest().(*Message_LotteryRequest); ok {
return x.LotteryRequest
@ -331,7 +331,7 @@ type isMessage_Request interface {
}
type Message_Staking struct {
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Staking *StakingRequest `protobuf:"bytes,4,opt,name=staking,proto3,oneof"`
}
@ -340,7 +340,7 @@ type Message_Consensus struct {
}
type Message_Drand struct {
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Drand *DrandRequest `protobuf:"bytes,6,opt,name=drand,proto3,oneof"`
}
@ -351,7 +351,7 @@ type Message_Viewchange struct {
type Message_LotteryRequest struct {
// Refactor this later after demo.
//
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
LotteryRequest *LotteryRequest `protobuf:"bytes,8,opt,name=lottery_request,json=lotteryRequest,proto3,oneof"`
}
@ -431,7 +431,7 @@ func (m *Response) GetResponse() isResponse_Response {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *Response) GetLotteryResponse() *LotteryResponse {
if x, ok := x.GetResponse().(*Response_LotteryResponse); ok {
return x.LotteryResponse
@ -444,7 +444,7 @@ type isResponse_Response interface {
}
type Response_LotteryResponse struct {
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
LotteryResponse *LotteryResponse `protobuf:"bytes,3,opt,name=lottery_response,json=lotteryResponse,proto3,oneof"`
}
@ -455,9 +455,9 @@ type LotteryResponse struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Players []string `protobuf:"bytes,2,rep,name=players,proto3" json:"players,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Balances []string `protobuf:"bytes,3,rep,name=balances,proto3" json:"balances,omitempty"`
}
@ -493,7 +493,7 @@ func (*LotteryResponse) Descriptor() ([]byte, []int) {
return file_message_proto_rawDescGZIP(), []int{2}
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *LotteryResponse) GetPlayers() []string {
if x != nil {
return x.Players
@ -501,7 +501,7 @@ func (x *LotteryResponse) GetPlayers() []string {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *LotteryResponse) GetBalances() []string {
if x != nil {
return x.Balances
@ -514,11 +514,11 @@ type LotteryRequest struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Type LotteryRequest_Type `protobuf:"varint,1,opt,name=type,proto3,enum=message.LotteryRequest_Type" json:"type,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
PrivateKey string `protobuf:"bytes,2,opt,name=private_key,json=privateKey,proto3" json:"private_key,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Amount int64 `protobuf:"varint,3,opt,name=amount,proto3" json:"amount,omitempty"`
}
@ -554,7 +554,7 @@ func (*LotteryRequest) Descriptor() ([]byte, []int) {
return file_message_proto_rawDescGZIP(), []int{3}
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *LotteryRequest) GetType() LotteryRequest_Type {
if x != nil {
return x.Type
@ -562,7 +562,7 @@ func (x *LotteryRequest) GetType() LotteryRequest_Type {
return LotteryRequest_ENTER
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *LotteryRequest) GetPrivateKey() string {
if x != nil {
return x.PrivateKey
@ -570,7 +570,7 @@ func (x *LotteryRequest) GetPrivateKey() string {
return ""
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *LotteryRequest) GetAmount() int64 {
if x != nil {
return x.Amount
@ -584,9 +584,9 @@ type StakingRequest struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Transaction []byte `protobuf:"bytes,1,opt,name=transaction,proto3" json:"transaction,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
NodeId string `protobuf:"bytes,2,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
}
@ -622,7 +622,7 @@ func (*StakingRequest) Descriptor() ([]byte, []int) {
return file_message_proto_rawDescGZIP(), []int{4}
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *StakingRequest) GetTransaction() []byte {
if x != nil {
return x.Transaction
@ -630,7 +630,7 @@ func (x *StakingRequest) GetTransaction() []byte {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *StakingRequest) GetNodeId() string {
if x != nil {
return x.NodeId
@ -746,13 +746,13 @@ type DrandRequest struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
SenderPubkey []byte `protobuf:"bytes,2,opt,name=sender_pubkey,json=senderPubkey,proto3" json:"sender_pubkey,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
BlockHash []byte `protobuf:"bytes,3,opt,name=block_hash,json=blockHash,proto3" json:"block_hash,omitempty"`
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
Payload []byte `protobuf:"bytes,4,opt,name=payload,proto3" json:"payload,omitempty"`
}
@ -788,7 +788,7 @@ func (*DrandRequest) Descriptor() ([]byte, []int) {
return file_message_proto_rawDescGZIP(), []int{6}
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *DrandRequest) GetShardId() uint32 {
if x != nil {
return x.ShardId
@ -796,7 +796,7 @@ func (x *DrandRequest) GetShardId() uint32 {
return 0
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *DrandRequest) GetSenderPubkey() []byte {
if x != nil {
return x.SenderPubkey
@ -804,7 +804,7 @@ func (x *DrandRequest) GetSenderPubkey() []byte {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *DrandRequest) GetBlockHash() []byte {
if x != nil {
return x.BlockHash
@ -812,7 +812,7 @@ func (x *DrandRequest) GetBlockHash() []byte {
return nil
}
// Deprecated: Do not use.
// Deprecated: Marked as deprecated in message.proto.
func (x *DrandRequest) GetPayload() []byte {
if x != nil {
return x.Payload

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.12.4
// source: message.proto
package message
@ -14,6 +18,10 @@ import (
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
const (
ClientService_Process_FullMethodName = "/message.ClientService/Process"
)
// ClientServiceClient is the client API for ClientService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@ -31,7 +39,7 @@ func NewClientServiceClient(cc grpc.ClientConnInterface) ClientServiceClient {
func (c *clientServiceClient) Process(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Response, error) {
out := new(Response)
err := c.cc.Invoke(ctx, "/message.ClientService/Process", in, out, opts...)
err := c.cc.Invoke(ctx, ClientService_Process_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -76,7 +84,7 @@ func _ClientService_Process_Handler(srv interface{}, ctx context.Context, dec fu
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/message.ClientService/Process",
FullMethod: ClientService_Process_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ClientServiceServer).Process(ctx, req.(*Message))

@ -0,0 +1,9 @@
FROM golang:1.19-bullseye
RUN apt update
RUN apt install -y protobuf-compiler
RUN protoc --version
RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0
RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3
ENTRYPOINT ["protoc", "-I=/tmp", "--go_out=/tmp", "--go-grpc_out=/tmp"]

@ -0,0 +1,4 @@
# if command fails, try this
# docker buildx create --use
docker buildx build -t frozen621/harmony-proto:latest --platform linux/amd64,linux/arm64 -f Proto.Dockerfile --progress=plain .

@ -1,5 +0,0 @@
# used versions
#go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
#go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
SRC_DIR=$(dirname $0)
protoc -I ${SRC_DIR}/proto/ ${SRC_DIR}/proto/downloader.proto --go_out=${SRC_DIR}/proto --go-grpc_out=${SRC_DIR}/proto

@ -1,4 +1,5 @@
package downloader
///go:generate protoc downloader.proto --go_out=plugins=grpc:.
//go:generate protoc downloader.proto --go_out=. --go-grpc_out=.
///go:generate protoc downloader.proto --go_out=. --go-grpc_out=.
//go:generate ./gen.sh

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.15.8
// protoc-gen-go v1.30.0
// protoc v3.12.4
// source: downloader.proto
package downloader

@ -1,4 +1,8 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.12.4
// source: downloader.proto
package downloader
@ -14,6 +18,10 @@ import (
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
const (
Downloader_Query_FullMethodName = "/downloader.Downloader/Query"
)
// DownloaderClient is the client API for Downloader service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@ -31,7 +39,7 @@ func NewDownloaderClient(cc grpc.ClientConnInterface) DownloaderClient {
func (c *downloaderClient) Query(ctx context.Context, in *DownloaderRequest, opts ...grpc.CallOption) (*DownloaderResponse, error) {
out := new(DownloaderResponse)
err := c.cc.Invoke(ctx, "/downloader.Downloader/Query", in, out, opts...)
err := c.cc.Invoke(ctx, Downloader_Query_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -76,7 +84,7 @@ func _Downloader_Query_Handler(srv interface{}, ctx context.Context, dec func(in
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/downloader.Downloader/Query",
FullMethod: Downloader_Query_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DownloaderServer).Query(ctx, req.(*DownloaderRequest))

@ -0,0 +1,9 @@
#!/bin/bash
# used versions
#go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
#go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
#SRC_DIR=$(dirname $0)
#protoc -I ${SRC_DIR}/proto/ ${SRC_DIR}/proto/downloader.proto --go_out=${SRC_DIR}/proto --go-grpc_out=${SRC_DIR}/proto
docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/downloader.proto

@ -0,0 +1,3 @@
#!/bin/bash
docker run --platform linux/amd64 -v ${PWD}:/tmp ${PROTOC_IMAGE} /tmp/msg.proto

@ -1,3 +1,3 @@
package message
//go:generate protoc msg.proto --go_out=.
//go:generate ./gen.sh

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.15.8
// protoc-gen-go v1.30.0
// protoc v3.12.4
// source: msg.proto
package message

@ -6,4 +6,4 @@ case "${0}" in
*) progdir=.;;
esac
git grep -l '^//go:generate ' -- '*.go' | \
"${progdir}/xargs_by_dir.sh" go generate -v -x
PROTOC_IMAGE="frozen621/harmony-proto:latest" "${progdir}/xargs_by_dir.sh" go generate -v -x

@ -1,15 +0,0 @@
#!/bin/sh
set -eu
unset -v progdir
case "${0}" in
/*) progdir="/";;
*/*) progdir="${0%/*}";;
*) progdir=".";
esac
sed -n 's/^ _ "\([^"]*\)"$/\1/p' "${progdir}/../tools/tools.go" | \
xargs "${progdir}/goget.sh"
"${progdir}/install_protoc.sh" -V 3.15.8

@ -1,66 +0,0 @@
#!/usr/bin/env bash
set -eu
unset -v opt OPTIND OPTARG version platform destdir
version=
platform=
destdir=/usr/local
usage() {
case $# in
[1-9]*) echo "$@" >&2;;
esac
cat <<- ENDEND
usage: install_protoc.sh [-P platform] [-d destdir] -V version
options:
-V version protobuf version
-P platform fetch and use given platform (default: autodetect)
-d destdir install into the given dir (default: /usr/local)
-h print this help
ENDEND
exit 64
}
while getopts :V:P:d:h opt
do
case "${opt}" in
V) version="${OPTARG}";;
P) platform="${OPTARG}";;
d) destdir="${OPTARG}";;
h) usage;;
"?") usage "unrecognized option -${OPTARG}";;
":") usage "missing argument for -${OPTARG}";;
*) echo "unhandled option -${OPTARG}" >&2; exit 70;;
esac
done
shift $((${OPTIND} - 1))
case "${version}" in
"") usage "protobuf version (-V) not specified";;
esac
case "${platform}" in
"")
platform=$(uname -s)
case "${platform}" in
Darwin) platform=osx;;
Linux) platform=linux;;
*) echo "unsupported OS name (${platform})" >&2; exit 69;;
esac
arch=$(uname -m)
case "${arch}" in
aarch64) arch=aarch_64;;
esac
platform="${platform}-${arch}"
;;
esac
unset -v tmpdir
trap 'case "${tmpdir-}" in ?*) rm -rf "${tmpdir}";; esac' EXIT
tmpdir=$(mktemp -d)
cd "${tmpdir}"
unset -v filename url
filename="protoc-${version}-${platform}.zip"
url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/${filename}"
echo "Downloading protoc v${version} for ${platform}..."
curl -s -S -L -o "${filename}" "${url}"
echo "Downloaded as ${filename}; unzipping into ${destdir}..."
sudo unzip -o -d "${destdir}" "${filename}"
echo "protoc v${version} has been installed in ${destdir}."
sudo chmod +x "${destdir}/bin/protoc"
exit 0
Loading…
Cancel
Save