Merge pull request #3313 from LeoHChen/publish_repo

Publish RPM/Deb repo
pull/3317/head
Leo Chen 4 years ago committed by GitHub
commit 1e126d2f51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      .rpmmacros
  2. 46
      Makefile
  3. 1
      scripts/package/deb/DEBIAN/control
  4. 17
      scripts/package/deb/DEBIAN/postinst
  5. 8
      scripts/package/deb/DEBIAN/postrm
  6. 2
      scripts/package/deb/DEBIAN/prerm
  7. 30
      scripts/package/deb/build.sh
  8. 1
      scripts/package/deb/changelog
  9. 42
      scripts/package/deb/dev.aptly.conf
  10. 42
      scripts/package/deb/prod.aptly.conf
  11. 67
      scripts/package/harmony-rclone.sh
  12. 17
      scripts/package/harmony-setup.sh
  13. 104
      scripts/package/publish-repo.sh
  14. 6
      scripts/package/rpm/harmony-dev.repo
  15. 6
      scripts/package/rpm/harmony-prod.repo
  16. 30
      scripts/package/rpm/harmony.spec

@ -1 +1,2 @@
%_topdir %(echo $HOME)/rpmbuild
%_gpg_name Leo Chen <leo@harmony.one>

@ -6,12 +6,13 @@ export LIBRARY_PATH:=$(LD_LIBRARY_PATH)
export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH)
export GO111MODULE:=on
PKGNAME=harmony
VERSION?=2.3.5
VERSION?=$(shell git tag -l --sort=-v:refname | head -n 1 | tr -d v)
RELEASE?=$(shell git describe --long | cut -f2 -d-)
RPMBUILD=$(HOME)/rpmbuild
DEBBUILD=$(HOME)/debbuild
SHELL := bash
.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb rpm_init rpm_build rpm
.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod
all: libs
bash ./scripts/go_executable_build.sh -S
@ -32,7 +33,7 @@ help:
@echo "linux_static - static build the harmony binary & bootnode along with the MCL & BLS libs (for linux)"
@echo "arm_static - static build the harmony binary & bootnode on ARM64 platform"
@echo "rpm - build a harmony RPM pacakge"
@echo "deb - build a harmony Debian pacakge (todo)"
@echo "deb - build a harmony Debian pacakge"
libs:
make -C $(TOP)/mcl -j8
@ -88,22 +89,30 @@ arm_static:
deb_init:
rm -rf $(DEBBUILD)
mkdir -p $(DEBBUILD)/$(PKGNAME)-$(VERSION)/{etc/systemd/system,usr/sbin,etc/sysctl.d,etc/harmony}
cp -f bin/harmony $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/
bin/harmony dumpconfig $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/harmony/harmony.conf
cp -f scripts/package/rclone.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/harmony/
cp -f scripts/package/harmony.service $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/systemd/system/
cp -f scripts/package/harmony-setup.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/
cp -f scripts/package/harmony-rclone.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/
cp -f scripts/package/harmony-sysctl.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/sysctl.d/99-harmony.conf
cp -r scripts/package/deb/DEBIAN $(DEBBUILD)/$(PKGNAME)-$(VERSION)
VER=$(VERSION) scripts/package/templater.sh scripts/package/deb/DEBIAN/control > $(DEBBUILD)/$(PKGNAME)-$(VERSION)/DEBIAN/control
mkdir -p $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/{etc/systemd/system,usr/sbin,etc/sysctl.d,etc/harmony}
cp -f bin/harmony $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/
bin/harmony dumpconfig $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/harmony/harmony.conf
cp -f scripts/package/rclone.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/harmony/
cp -f scripts/package/harmony.service $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/systemd/system/
cp -f scripts/package/harmony-setup.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/
cp -f scripts/package/harmony-rclone.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/
cp -f scripts/package/harmony-sysctl.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/sysctl.d/99-harmony.conf
cp -r scripts/package/deb/DEBIAN $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)
VER=$(VERSION)-$(RELEASE) scripts/package/templater.sh scripts/package/deb/DEBIAN/control > $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/DEBIAN/control
deb_build:
(cd $(DEBBUILD); dpkg-deb --build $(PKGNAME)-$(VERSION)/)
(cd $(DEBBUILD); dpkg-deb --build $(PKGNAME)-$(VERSION)-$(RELEASE)/)
deb: deb_init deb_build
debpub_dev: deb
cp scripts/package/deb/dev.aptly.conf ~/.aptly.conf
./scripts/package/publish-repo.sh -p dev -n deb -s $(DEBBUILD)
debpub_prod: deb
cp scripts/package/deb/prod.aptly.conf ~/.aptly.conf
./scripts/package/publish-repo.sh -p prod -n deb -s $(DEBBUILD)
rpm_init:
rm -rf $(RPMBUILD)
mkdir -p $(RPMBUILD)/{SOURCES,SPECS,BUILD,RPMS,BUILDROOT,SRPMS}
@ -115,10 +124,17 @@ rpm_init:
cp -f scripts/package/harmony-rclone.sh $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION)
cp -f scripts/package/rclone.conf $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION)
cp -f scripts/package/harmony-sysctl.conf $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION)
VER=$(VERSION) scripts/package/templater.sh scripts/package/rpm/harmony.spec > $(RPMBUILD)/SPECS/harmony.spec
VER=$(VERSION) REL=$(RELEASE) scripts/package/templater.sh scripts/package/rpm/harmony.spec > $(RPMBUILD)/SPECS/harmony.spec
(cd $(RPMBUILD)/SOURCES; tar cvf $(PKGNAME)-$(VERSION).tar $(PKGNAME)-$(VERSION))
rpm_build:
rpmbuild --target x86_64 -bb $(RPMBUILD)/SPECS/harmony.spec
rpm: rpm_init rpm_build
rpm --addsign $(RPMBUILD)/RPMS/x86_64/$(PKGNAME)-$(VERSION)-$(RELEASE).x86_64.rpm
rpmpub_dev: rpm
./scripts/package/publish-repo.sh -p dev -n rpm -s $(RPMBUILD)
rpmpub_prod: rpm
./scripts/package/publish-repo.sh -p prod -n rpm -s $(RPMBUILD)

@ -11,3 +11,4 @@ Priority: extra
Homepage: https://harmony.one
Description: Harmony is a sharded, fast finality, low fee, PoS public blockchain. This package contains the validator node program for harmony blockchain.
Build-Dep: dh-systemd
Recommends: jq, rclone (>= 1.52.3)

@ -1,20 +1,23 @@
#!/bin/sh
set -e
after_install() {
:
mkdir -p /home/harmony/.hmy/blskeys
mkdir -p /home/harmony/.config/rclone
chown -R harmony.harmony /home/harmony
systemctl enable harmony
systemctl start harmony
}
after_upgrade() {
:
# systemctl restart harmony
systemctl enable harmony
systemctl restart harmony
}
# Setup system users and groups
addgroup --quiet --system harmony
adduser --quiet --system --ingroup harmony --gecos "harmony validator account" harmony
getent group harmony >/dev/null || addgroup --quiet --system harmony
getent passwd harmony >/dev/null || adduser --quiet --system --ingroup harmony --gecos "harmony validator account" harmony
if [ "${1}" = "configure" -a -z "${2}" ] || \
[ "${1}" = "abort-remove" ]

@ -1,4 +1,7 @@
#!/bin/sh
set -e
after_remove() {
:
#rm -rf /data/harmony/latest
@ -7,7 +10,10 @@ after_remove() {
}
after_purge() {
:
systemctl disable harmony
userdel harmony || true
groupdel harmony || true
rm -rf /etc/harmony
}
dummy() {

@ -3,7 +3,7 @@
set -e
before_remove() {
:
systemctl stop harmony
}
dummy() {

@ -1,30 +0,0 @@
#!/bin/bash
if [ $# != 1 ]; then
echo "$0 version"
exit
fi
VERSION=$1
fpm -s dir -t deb -n harmony -p harmony_VERSION_ARCH.deb -C harmony-${VERSION} \
-v "$VERSION" \
--license "MIT" \
--vendor "Harmony Blockchain" \
--category "net" \
--no-depends \
--no-auto-depends \
--directories /etc/harmony \
--directories /data/harmony \
--architecture x86_64 \
--maintainer "Leo Chen <leo@harmony.one>" \
--description "Harmony is a sharded, fast finality, low fee, PoS public blockchain.\nThis package contains the validator node program for harmony blockchain." \
--url "https://harmony.one" \
--before-install scripts/preinst \
--after-install scripts/postinst \
--before-remove scripts/prerm \
--after-remove scripts/postrm \
--before-upgrade scripts/preup \
--after-upgrade scripts/postup \
--deb-changelog scripts/changelog \
--deb-systemd-restart-after-upgrade

@ -1 +1,2 @@
1.1 improved debian scripts
1.0 init harmony debian package

@ -0,0 +1,42 @@
{
"rootDir": "/tmp/.aptly",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"FileSystemPublishEndpoints": {
"harmony-dev": {
"rootDir": "/tmp/repo",
"linkMethod": "copy",
"verifyMethod": "md5"
}
},
"S3PublishEndpoints": {
"harmony-dev": {
"region": "us-east-1",
"bucket": "haochen-harmony-pub",
"endpoint": "",
"awsAccessKeyID": "",
"awsSecretAccessKey": "",
"prefix": "pub/repo",
"acl": "public-read",
"storageClass": "",
"encryptionMethod": "",
"plusWorkaround": false,
"disableMultiDel": false,
"forceSigV2": false,
"debug": false
}
}
}

@ -0,0 +1,42 @@
{
"rootDir": "/tmp/.aptly",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": false,
"gpgDisableVerify": false,
"gpgProvider": "gpg",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"FileSystemPublishEndpoints": {
"harmony-prod": {
"rootDir": "/tmp/repo",
"linkMethod": "copy",
"verifyMethod": "md5"
}
},
"S3PublishEndpoints": {
"harmony-prod": {
"region": "us-west-1",
"bucket": "pub.harmony.one",
"endpoint": "",
"awsAccessKeyID": "",
"awsSecretAccessKey": "",
"prefix": "release/package/apt",
"acl": "public-read",
"storageClass": "",
"encryptionMethod": "",
"plusWorkaround": false,
"disableMultiDel": false,
"forceSigV2": false,
"debug": false
}
}
}

@ -1,13 +1,64 @@
#!/bin/bash
if [ $# != 1 ]; then
echo "usage: $0 datadir"
exit 1
fi
ME=$(basename "$0")
function usage() {
local MSG=$1
cat<<-EOT
$MSG
This script will rclone the harmony db to datadir/archive directory.
Usage: $ME [options] datadir shard
datadir: the root directory of the harmony db (default: /home/harmony)
shard: the shard number to sync (valid value: 0,1,2,3)
HARMONY_HOME="$1"
Options:
-h print this help message
-c clean up backup db after rclone
-a sync archival db, instead of regular db
sudo -u harmony mkdir -p ${HARMONY_HOME}/archive
for i in 0 1 2 3; do
sudo -u harmony rclone sync -vvv hmy:pub.harmony.one/mainnet.min/harmony_db_$i ${HARMONY_HOME}/archive/harmony_db_$i > ${HARMONY_HOME}/archive/archive-$i.log 2>&1
EOT
exit 1
}
CLEAN=false
FOLDER=mainnet.min
while getopts ":hca" opt; do
case $opt in
c) CLEAN=true ;;
a) FOLDER=mainnet.archival ;;
*) usage ;;
esac
done
shift $((OPTIND - 1))
if [ $# != 2 ]; then
usage
fi
DATADIR="$1"
SHARD="$2"
if [ ! -d "$DATADIR" ]; then
usage "ERROR: no datadir directory: $DATADIR"
fi
case "$SHARD" in
0|1|2|3) ;;
*) usage "ERROR: invalid shard number: $SHARD" ;;
esac
mkdir -p "${DATADIR}/archive"
rclone sync -vvv "hmy:pub.harmony.one/${FOLDER}/harmony_db_${SHARD}" "${DATADIR}/archive/harmony_db_${SHARD}" > "${DATADIR}/archive/archive-${SHARD}.log" 2>&1
[ -d "${DATADIR}/harmony_db_${SHARD}" ] && mv -f "${DATADIR}/harmony_db_${SHARD}" "${DATADIR}/archive/harmony_db_${SHARD}.bak"
mv -f "${DATADIR}/archive/harmony_db_${SHARD}" "${DATADIR}/harmony_db_${SHARD}"
if $CLEAN; then
rm -rf "${DATADIR}/archive/harmony_db_${SHARD}.bak"
fi

@ -5,7 +5,12 @@ CONFIG=/etc/harmony/harmony.conf
VER=v1.0
function usage() {
local MSG=${1}
cat <<-EOT
$MSG
This script is a helper script to edit the $CONFIG.
Usage: $ME [options]
Options:
@ -23,18 +28,18 @@ EOT
}
function _setup_validator_config_file() {
sed -i.bak 's,NodeType =.*,NodeType = "validator",; s,ShardID = .*,ShardID = -1,' $CONFIG
sed -i.bak 's,NodeType =.*,NodeType = "validator",; s,ShardID = .*,ShardID = -1,; s,IsArchival = .*,IsArchival = false,' $CONFIG
}
function _setup_explorer_config_file() {
sed -i.bak "s,NodeType =.*,NodeType = \"explorer\",; s,ShardID = .*,ShardID = $SHARD," $CONFIG
sed -i.bak "s,NodeType =.*,NodeType = \"explorer\",; s,ShardID = .*,ShardID = $SHARD,; s,IsArchival = .*,IsArchival = true," $CONFIG
}
function setup_config_file() {
case $TYPE in
validator) _setup_validator_config_file ;;
explorer) _setup_explorer_config_file ;;
*) usage ;;
*) usage "ERROR: invalid node type! '$TYPE'" ;;
esac
}
@ -58,16 +63,16 @@ case ${TYPE} in
explorer)
case ${SHARD} in
0|1|2|3) ;;
*) usage ;;
*) usage "ERROR: invalid shard number! '$SHARD'" ;;
esac
;;
validator)
case ${SHARD} in
-1) ;;
*) usage ;;
*) usage "ERROR: do not specify shard number in validator type!!" ;;
esac
;;
*) usage ;;
*) usage "ERROR: invalid node type! '$TYPE'" ;;
esac
setup_config_file

@ -0,0 +1,104 @@
#!/usr/bin/env bash
ME=$(basename "$0")
DEPS=(aws createrepo aptly gpg)
PKG=rpm
PROFILE=dev
SRC=
# destination of the bucket to host the repo
declare -A TARGET
TARGET[rpm.dev]="haochen-harmony-pub/pub/yum"
TARGET[deb.dev]="haochen-harmony-pub/pub/repo"
TARGET[rpm.prod]="pub.harmony.one/release/package/yum"
TARGET[deb.prod]="pub.harmony.one/release/package/apt"
function usage() {
cat<<-EOT
Usage: $ME [options]
Option:
-h print this help message
-p dev/prod profile of the repo (dev/prod, default: $PROFILE)
-n rpm/deb type of package for publish (rpm/deb, default: $PKG)
-s directory source of the package repo
Examples:
$ME -p dev -n rpm -s ~/rpmbuild
$ME -p prod -n deb -s ~/debbuild
EOT
exit 0
}
function validation() {
for dep in "${DEPS[@]}"; do
if ! command -v "$dep" > /dev/null; then
echo "missing dependency: $dep"
exit 1
fi
done
case $PROFILE in
dev|prod) ;;
*) usage ;;
esac
if [[ -z "$SRC" || ! -d "$SRC" ]]; then
echo "missing source path or wrong path: $SRC"
exit 1
fi
}
function publish_rpm() {
local target
local tempdir
target=${TARGET[$PKG.$PROFILE]}
tempdir="/tmp/$(basename $target)"
mkdir -p "$tempdir/x86_64"
aws s3 sync "s3://$target" "$tempdir"
cp -rv $SRC/RPMS/x86_64/* "$tempdir/x86_64"
UPDATE=""
if [ -e "$tempdir/x86_64/repodata/repomd.xml" ]; then
UPDATE="--update"
fi
createrepo -v $UPDATE --deltas "$tempdir/x86_64/"
aws s3 sync "$tempdir" "s3://$target" --acl public-read
}
function publish_deb() {
if aptly repo show harmony-$PROFILE > /dev/null; then
aptly repo add harmony-$PROFILE $SRC
aptly publish update bionic s3:harmony-$PROFILE:
else
aptly repo create -distribution=bionic -component=main harmony-$PROFILE
aptly repo add harmony-$PROFILE $SRC
aptly publish repo harmony-$PROFILE s3:harmony-$PROFILE:
fi
}
################## MAIN ##################
if [ $# = 0 ]; then
usage
fi
while getopts ":hp:n:s:" opt; do
case $opt in
p) PROFILE=${OPTARG} ;;
n) PKG=${OPTARG} ;;
s) SRC=${OPTARG} ;;
*) usage ;;
esac
done
shift $((OPTIND - 1))
validation
case $PKG in
rpm) publish_rpm ;;
deb) publish_deb ;;
*) usage ;;
esac

@ -0,0 +1,6 @@
[harmony-dev]
name=Development Packages - $basearch
baseurl=http://haochen-harmony-pub.s3.amazonaws.com/pub/yum/$basearch/
enabled=1
gpgkey=https://raw.githubusercontent.com/harmony-one/harmony-open/master/harmony-release/harmony-pub.key
gpgcheck=1

@ -0,0 +1,6 @@
[harmony-prod]
name=Production Packages - $basearch
baseurl=http://pub.harmony.one.s3.amazonaws.com/release/package/yum/$basearch/
enabled=1
gpgkey=https://raw.githubusercontent.com/harmony-one/harmony-open/master/harmony-release/harmony-pub.key
gpgcheck=1

@ -1,4 +1,5 @@
{{VER=2.0.0}}
{{REL=0}}
# SPEC file overview:
# https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages/#con_rpm-spec-file-overview
# Fedora packaging guidelines:
@ -6,16 +7,16 @@
Name: harmony
Version: {{ VER }}
Release: 0
Release: {{ REL }}
Summary: harmony blockchain validator node program
License: MIT
URL: https://harmony.one
Source0: %{name}-%{version}.tar
BuildArch: x86_64
Packager: Leo Chen
Packager: Leo Chen <leo@hamrony.one>
Requires(pre): shadow-utils
Requires: systemd-rpm-macros
Requires: systemd-rpm-macros jq
%description
Harmony is a sharded, fast finality, low fee, PoS public blockchain.
@ -57,14 +58,17 @@ install -m 0644 harmony.conf ${RPM_BUILD_ROOT}/etc/harmony/
exit 0
%post
%systemd_post %{name}.service
%systemd_user_post %{name}.service
%sysctl_apply %{name}-sysctl.conf
exit 0
%preun
%systemd_preun %{name}.service
%systemd_user_preun %{name}.service
exit 0
%postun
%systemd_postun_with_restart ${name}.service
%systemd_postun_with_restart %{name}.service
exit 0
%files
/usr/sbin/harmony
@ -76,11 +80,21 @@ exit 0
/etc/harmony/rclone.conf
/home/harmony/.config/rclone
%config(noreplace) /etc/harmony/harmony.conf
%config /etc/harmony/rclone.conf
%config /etc/sysctl.d/99-harmony.conf
%config /etc/systemd/system/harmony.service
%doc
%license
%changelog
* Tue Aug 18 2020 Leo Chen <leo at harmony dot one> 2.3.5
- init version of the harmony node program
* Wed Aug 26 2020 Leo Chen <leo at harmony dot one> 2.3.5
- get version from git tag
- add %config macro to keep edited config files
* Tue Aug 18 2020 Leo Chen <leo at harmony dot one> 2.3.4
- init version of the harmony node program

Loading…
Cancel
Save