From 57343d8e0e96b5ef019c3e7d6267c77b108124da Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Sun, 23 Aug 2020 06:17:42 +0000 Subject: [PATCH 1/9] [rpm] support rpm signing [rpm] publish rpm package to yum repo Signed-off-by: Leo Chen --- .rpmmacros | 1 + Makefile | 7 ++ scripts/package/publish-repo.sh | 97 +++++++++++++++++++++++++++ scripts/package/rpm/harmony-dev.repo | 6 ++ scripts/package/rpm/harmony-prod.repo | 6 ++ 5 files changed, 117 insertions(+) create mode 100755 scripts/package/publish-repo.sh create mode 100644 scripts/package/rpm/harmony-dev.repo create mode 100644 scripts/package/rpm/harmony-prod.repo diff --git a/.rpmmacros b/.rpmmacros index 1af8248c6..701da7cb8 100644 --- a/.rpmmacros +++ b/.rpmmacros @@ -1 +1,2 @@ %_topdir %(echo $HOME)/rpmbuild +%_gpg_name Leo Chen diff --git a/Makefile b/Makefile index 122904c53..9507d2bd6 100644 --- a/Makefile +++ b/Makefile @@ -122,3 +122,10 @@ rpm_build: rpmbuild --target x86_64 -bb $(RPMBUILD)/SPECS/harmony.spec rpm: rpm_init rpm_build + rpm --addsign $(RPMBUILD)/RPMS/x86_64/$(PKGNAME)-$(VERSION)-0.x86_64.rpm + +rpmpub_dev: rpm + ./scripts/package/publish-repo.sh -p dev -t rpm -s $(RPMBUILD) + +rpmpub_prod: rpm + ./scripts/package/publish-repo.sh -p prod -t rpm -s $(RPMBUILD) diff --git a/scripts/package/publish-repo.sh b/scripts/package/publish-repo.sh new file mode 100755 index 000000000..b93af4731 --- /dev/null +++ b/scripts/package/publish-repo.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +ME=$(basename "$0") +DEPS=(aws createrepo aptly gpg) +PKG=rpm +PROFILE=dev +SRC= + +# destination of the bucket to host the repo +declare -A TARGET +TARGET[rpm.dev]="haochen-harmony-pub/pub/yum" +TARGET[deb.dev]="haochen-harmony-pub/pub/repo" +TARGET[rpm.prod]="pub.harmony.one/release/package/yum" +TARGET[deb.prod]="pub.harmony.one/release/package/apt" + +function usage() { + cat<<-EOT +Usage: $ME [options] + +Option: + -h print this help message + -p dev/prod profile of the repo (dev/prod, default: $PROFILE) + -n rpm/deb type of package for publish (rpm/deb, default: $PKG) + -s directory source of the package repo + +Examples: + $ME -p dev -n rpm -s ~/rpmbuild + + $ME -p prod -n deb -s ~/debbuild + +EOT + exit 0 +} + +function validation() { + for dep in "${DEPS[@]}"; do + if ! command -v "$dep" > /dev/null; then + echo "missing dependency: $dep" + exit 1 + fi + done + + case $PROFILE in + dev|prod) ;; + *) usage ;; + esac + + if [[ -z "$SRC" || ! -d "$SRC" ]]; then + echo "missing source path or wrong path: $SRC" + exit 1 + fi +} + +function publish_rpm() { + local target + local tempdir + target=${TARGET[$PKG.$PROFILE]} + tempdir="/tmp/$(basename $target)" + mkdir -p "$tempdir/x86_64" + aws s3 sync "s3://$target" "$tempdir" + cp -rv $SRC/RPMS/x86_64/* "$tempdir/x86_64" + UPDATE="" + if [ -e "$tempdir/x86_64/repodata/repomd.xml" ]; then + UPDATE="--update" + fi + createrepo -v $UPDATE --deltas "$tempdir/x86_64/" + + aws s3 sync "$tempdir" "s3://$target" --acl public-read +} + +function publish_deb() { + : +} + +################## MAIN ################## +if [ $# = 0 ]; then + usage +fi + +while getopts ":hp:n:s:" opt; do + case $opt in + p) PROFILE=${OPTARG} ;; + n) PKG=${OPTARG} ;; + s) SRC=${OPTARG} ;; + *) usage ;; + esac +done + +shift $((OPTIND - 1)) + +validation + +case $PKG in + rpm) publish_rpm ;; + deb) publish_deb ;; + *) usage ;; +esac diff --git a/scripts/package/rpm/harmony-dev.repo b/scripts/package/rpm/harmony-dev.repo new file mode 100644 index 000000000..ffa82a5b6 --- /dev/null +++ b/scripts/package/rpm/harmony-dev.repo @@ -0,0 +1,6 @@ +[harmony-dev] +name=Development Packages - $basearch +baseurl=http://haochen-harmony-pub.s3.amazonaws.com/pub/yum/$basearch/ +enabled=1 +gpgkey=https://raw.githubusercontent.com/harmony-one/harmony-open/master/harmony-release/harmony-pub.key +gpgcheck=1 diff --git a/scripts/package/rpm/harmony-prod.repo b/scripts/package/rpm/harmony-prod.repo new file mode 100644 index 000000000..15e32e942 --- /dev/null +++ b/scripts/package/rpm/harmony-prod.repo @@ -0,0 +1,6 @@ +[harmony-prod] +name=Production Packages - $basearch +baseurl=http://pub.harmony.one.s3.amazonaws.com/release/package/yum/$basearch/ +enabled=1 +gpgkey=https://raw.githubusercontent.com/harmony-one/harmony-open/master/harmony-release/harmony-pub.key +gpgcheck=1 From b32c489ab3731ccad330b11d156a30ff9a286e1f Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Sun, 23 Aug 2020 10:09:02 +0000 Subject: [PATCH 2/9] [deb] publish deb package Signed-off-by: Leo Chen --- Makefile | 18 +++++++++---- scripts/package/deb/build.sh | 30 --------------------- scripts/package/deb/dev.aptly.conf | 42 +++++++++++++++++++++++++++++ scripts/package/deb/prod.aptly.conf | 42 +++++++++++++++++++++++++++++ scripts/package/publish-repo.sh | 11 +++++++- 5 files changed, 107 insertions(+), 36 deletions(-) delete mode 100755 scripts/package/deb/build.sh create mode 100644 scripts/package/deb/dev.aptly.conf create mode 100644 scripts/package/deb/prod.aptly.conf diff --git a/Makefile b/Makefile index 9507d2bd6..26bdb7e00 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,12 @@ export LIBRARY_PATH:=$(LD_LIBRARY_PATH) export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH) export GO111MODULE:=on PKGNAME=harmony -VERSION?=2.3.5 +VERSION?=2.3.6 RPMBUILD=$(HOME)/rpmbuild DEBBUILD=$(HOME)/debbuild SHELL := bash -.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb rpm_init rpm_build rpm +.PHONY: all help libs exe race trace-pointer debug debug-kill test test-go test-api test-api-attach linux_static deb_init deb_build deb debpub_dev debpub_prod rpm_init rpm_build rpm rpmpub_dev rpmpub_prod all: libs bash ./scripts/go_executable_build.sh -S @@ -32,7 +32,7 @@ help: @echo "linux_static - static build the harmony binary & bootnode along with the MCL & BLS libs (for linux)" @echo "arm_static - static build the harmony binary & bootnode on ARM64 platform" @echo "rpm - build a harmony RPM pacakge" - @echo "deb - build a harmony Debian pacakge (todo)" + @echo "deb - build a harmony Debian pacakge" libs: make -C $(TOP)/mcl -j8 @@ -104,6 +104,14 @@ deb_build: deb: deb_init deb_build +debpub_dev: deb + cp scripts/package/deb/dev.aptly.conf ~/.aptly.conf + ./scripts/package/publish-repo.sh -p dev -n deb -s $(DEBBUILD) + +debpub_prod: deb + cp scripts/package/deb/prod.aptly.conf ~/.aptly.conf + ./scripts/package/publish-repo.sh -p prod -n deb -s $(DEBBUILD) + rpm_init: rm -rf $(RPMBUILD) mkdir -p $(RPMBUILD)/{SOURCES,SPECS,BUILD,RPMS,BUILDROOT,SRPMS} @@ -125,7 +133,7 @@ rpm: rpm_init rpm_build rpm --addsign $(RPMBUILD)/RPMS/x86_64/$(PKGNAME)-$(VERSION)-0.x86_64.rpm rpmpub_dev: rpm - ./scripts/package/publish-repo.sh -p dev -t rpm -s $(RPMBUILD) + ./scripts/package/publish-repo.sh -p dev -n rpm -s $(RPMBUILD) rpmpub_prod: rpm - ./scripts/package/publish-repo.sh -p prod -t rpm -s $(RPMBUILD) + ./scripts/package/publish-repo.sh -p prod -n rpm -s $(RPMBUILD) diff --git a/scripts/package/deb/build.sh b/scripts/package/deb/build.sh deleted file mode 100755 index e81795c53..000000000 --- a/scripts/package/deb/build.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -if [ $# != 1 ]; then - echo "$0 version" - exit -fi - -VERSION=$1 - -fpm -s dir -t deb -n harmony -p harmony_VERSION_ARCH.deb -C harmony-${VERSION} \ - -v "$VERSION" \ - --license "MIT" \ - --vendor "Harmony Blockchain" \ - --category "net" \ - --no-depends \ - --no-auto-depends \ - --directories /etc/harmony \ - --directories /data/harmony \ - --architecture x86_64 \ - --maintainer "Leo Chen " \ - --description "Harmony is a sharded, fast finality, low fee, PoS public blockchain.\nThis package contains the validator node program for harmony blockchain." \ - --url "https://harmony.one" \ - --before-install scripts/preinst \ - --after-install scripts/postinst \ - --before-remove scripts/prerm \ - --after-remove scripts/postrm \ - --before-upgrade scripts/preup \ - --after-upgrade scripts/postup \ - --deb-changelog scripts/changelog \ - --deb-systemd-restart-after-upgrade diff --git a/scripts/package/deb/dev.aptly.conf b/scripts/package/deb/dev.aptly.conf new file mode 100644 index 000000000..74493c6e6 --- /dev/null +++ b/scripts/package/deb/dev.aptly.conf @@ -0,0 +1,42 @@ +{ + "rootDir": "/tmp/.aptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "FileSystemPublishEndpoints": { + "harmony-dev": { + "rootDir": "/tmp/repo", + "linkMethod": "copy", + "verifyMethod": "md5" + } + }, + "S3PublishEndpoints": { + "harmony-dev": { + "region": "us-east-1", + "bucket": "haochen-harmony-pub", + "endpoint": "", + "awsAccessKeyID": "", + "awsSecretAccessKey": "", + "prefix": "pub/repo", + "acl": "public-read", + "storageClass": "", + "encryptionMethod": "", + "plusWorkaround": false, + "disableMultiDel": false, + "forceSigV2": false, + "debug": false + } + } +} diff --git a/scripts/package/deb/prod.aptly.conf b/scripts/package/deb/prod.aptly.conf new file mode 100644 index 000000000..7f58394bd --- /dev/null +++ b/scripts/package/deb/prod.aptly.conf @@ -0,0 +1,42 @@ +{ + "rootDir": "/tmp/.aptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": false, + "gpgDisableVerify": false, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "FileSystemPublishEndpoints": { + "harmony-prod": { + "rootDir": "/tmp/repo", + "linkMethod": "copy", + "verifyMethod": "md5" + } + }, + "S3PublishEndpoints": { + "harmony-prod": { + "region": "us-west-1", + "bucket": "pub.harmony.one", + "endpoint": "", + "awsAccessKeyID": "", + "awsSecretAccessKey": "", + "prefix": "release/package/apt", + "acl": "public-read", + "storageClass": "", + "encryptionMethod": "", + "plusWorkaround": false, + "disableMultiDel": false, + "forceSigV2": false, + "debug": false + } + } +} diff --git a/scripts/package/publish-repo.sh b/scripts/package/publish-repo.sh index b93af4731..795894d3b 100755 --- a/scripts/package/publish-repo.sh +++ b/scripts/package/publish-repo.sh @@ -69,7 +69,16 @@ function publish_rpm() { } function publish_deb() { - : + if aptly repo show harmony-$PROFILE > /dev/null; then + aptly repo add harmony-$PROFILE $SRC + aptly publish update bionic s3:harmony-$PROFILE: + else + aptly repo create -distribution=bionic -component=main harmony-$PROFILE + aptly repo add harmony-$PROFILE $SRC + aptly publish repo harmony-$PROFILE s3:harmony-$PROFILE: + fi + # remove the local repo + aptly repo drop harmony-$PROFILE } ################## MAIN ################## From 0e1e6f38d65920827a86682fb0ef0d658eb835f4 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 06:00:57 +0000 Subject: [PATCH 3/9] [rpm/deb] query version/release from git tag Signed-off-by: Leo Chen --- Makefile | 29 +++++++++++++++-------------- scripts/package/rpm/harmony.spec | 5 +++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 26bdb7e00..1b289e31b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ export LIBRARY_PATH:=$(LD_LIBRARY_PATH) export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH) export GO111MODULE:=on PKGNAME=harmony -VERSION?=2.3.6 +VERSION?=$(shell git tag -l --sort=-v:refname | head -n 1 | tr -d v) +RELEASE?=$(shell git describe | cut -f2 -d-) RPMBUILD=$(HOME)/rpmbuild DEBBUILD=$(HOME)/debbuild SHELL := bash @@ -88,19 +89,19 @@ arm_static: deb_init: rm -rf $(DEBBUILD) - mkdir -p $(DEBBUILD)/$(PKGNAME)-$(VERSION)/{etc/systemd/system,usr/sbin,etc/sysctl.d,etc/harmony} - cp -f bin/harmony $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/ - bin/harmony dumpconfig $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/harmony/harmony.conf - cp -f scripts/package/rclone.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/harmony/ - cp -f scripts/package/harmony.service $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/systemd/system/ - cp -f scripts/package/harmony-setup.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/ - cp -f scripts/package/harmony-rclone.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)/usr/sbin/ - cp -f scripts/package/harmony-sysctl.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)/etc/sysctl.d/99-harmony.conf - cp -r scripts/package/deb/DEBIAN $(DEBBUILD)/$(PKGNAME)-$(VERSION) - VER=$(VERSION) scripts/package/templater.sh scripts/package/deb/DEBIAN/control > $(DEBBUILD)/$(PKGNAME)-$(VERSION)/DEBIAN/control + mkdir -p $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/{etc/systemd/system,usr/sbin,etc/sysctl.d,etc/harmony} + cp -f bin/harmony $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/ + bin/harmony dumpconfig $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/harmony/harmony.conf + cp -f scripts/package/rclone.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/harmony/ + cp -f scripts/package/harmony.service $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/systemd/system/ + cp -f scripts/package/harmony-setup.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/ + cp -f scripts/package/harmony-rclone.sh $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/usr/sbin/ + cp -f scripts/package/harmony-sysctl.conf $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/etc/sysctl.d/99-harmony.conf + cp -r scripts/package/deb/DEBIAN $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE) + VER=$(VERSION)-$(RELEASE) scripts/package/templater.sh scripts/package/deb/DEBIAN/control > $(DEBBUILD)/$(PKGNAME)-$(VERSION)-$(RELEASE)/DEBIAN/control deb_build: - (cd $(DEBBUILD); dpkg-deb --build $(PKGNAME)-$(VERSION)/) + (cd $(DEBBUILD); dpkg-deb --build $(PKGNAME)-$(VERSION)-$(RELEASE)/) deb: deb_init deb_build @@ -123,14 +124,14 @@ rpm_init: cp -f scripts/package/harmony-rclone.sh $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION) cp -f scripts/package/rclone.conf $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION) cp -f scripts/package/harmony-sysctl.conf $(RPMBUILD)/SOURCES/$(PKGNAME)-$(VERSION) - VER=$(VERSION) scripts/package/templater.sh scripts/package/rpm/harmony.spec > $(RPMBUILD)/SPECS/harmony.spec + VER=$(VERSION) REL=$(RELEASE) scripts/package/templater.sh scripts/package/rpm/harmony.spec > $(RPMBUILD)/SPECS/harmony.spec (cd $(RPMBUILD)/SOURCES; tar cvf $(PKGNAME)-$(VERSION).tar $(PKGNAME)-$(VERSION)) rpm_build: rpmbuild --target x86_64 -bb $(RPMBUILD)/SPECS/harmony.spec rpm: rpm_init rpm_build - rpm --addsign $(RPMBUILD)/RPMS/x86_64/$(PKGNAME)-$(VERSION)-0.x86_64.rpm + rpm --addsign $(RPMBUILD)/RPMS/x86_64/$(PKGNAME)-$(VERSION)-$(RELEASE).x86_64.rpm rpmpub_dev: rpm ./scripts/package/publish-repo.sh -p dev -n rpm -s $(RPMBUILD) diff --git a/scripts/package/rpm/harmony.spec b/scripts/package/rpm/harmony.spec index 757bf1fa6..9ff93c9b1 100644 --- a/scripts/package/rpm/harmony.spec +++ b/scripts/package/rpm/harmony.spec @@ -1,4 +1,5 @@ {{VER=2.0.0}} +{{REL=0}} # SPEC file overview: # https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages/#con_rpm-spec-file-overview # Fedora packaging guidelines: @@ -6,14 +7,14 @@ Name: harmony Version: {{ VER }} -Release: 0 +Release: {{ REL }} Summary: harmony blockchain validator node program License: MIT URL: https://harmony.one Source0: %{name}-%{version}.tar BuildArch: x86_64 -Packager: Leo Chen +Packager: Leo Chen Requires(pre): shadow-utils Requires: systemd-rpm-macros From 0c71a85bc83e498d0d8ee0dd17204761d11cccac Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 06:40:50 +0000 Subject: [PATCH 4/9] [harmony-setup] set IsArchival mode for validator/explorer Signed-off-by: Leo Chen --- scripts/package/harmony-setup.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/package/harmony-setup.sh b/scripts/package/harmony-setup.sh index c9588c4af..82057c4ad 100755 --- a/scripts/package/harmony-setup.sh +++ b/scripts/package/harmony-setup.sh @@ -5,7 +5,12 @@ CONFIG=/etc/harmony/harmony.conf VER=v1.0 function usage() { + local MSG=${1} + cat <<-EOT +$MSG +This script is a helper script to edit the $CONFIG. + Usage: $ME [options] Options: @@ -23,18 +28,18 @@ EOT } function _setup_validator_config_file() { - sed -i.bak 's,NodeType =.*,NodeType = "validator",; s,ShardID = .*,ShardID = -1,' $CONFIG + sed -i.bak 's,NodeType =.*,NodeType = "validator",; s,ShardID = .*,ShardID = -1,; s,IsArchival = .*,IsArchival = false,' $CONFIG } function _setup_explorer_config_file() { - sed -i.bak "s,NodeType =.*,NodeType = \"explorer\",; s,ShardID = .*,ShardID = $SHARD," $CONFIG + sed -i.bak "s,NodeType =.*,NodeType = \"explorer\",; s,ShardID = .*,ShardID = $SHARD,; s,IsArchival = .*,IsArchival = true," $CONFIG } function setup_config_file() { case $TYPE in validator) _setup_validator_config_file ;; explorer) _setup_explorer_config_file ;; - *) usage ;; + *) usage "ERROR: invalid node type! '$TYPE'" ;; esac } @@ -58,16 +63,16 @@ case ${TYPE} in explorer) case ${SHARD} in 0|1|2|3) ;; - *) usage ;; + *) usage "ERROR: invalid shard number! '$SHARD'" ;; esac ;; validator) case ${SHARD} in -1) ;; - *) usage ;; + *) usage "ERROR: do not specify shard number in validator type!!" ;; esac ;; - *) usage ;; + *) usage "ERROR: invalid node type! '$TYPE'" ;; esac setup_config_file From 9f378c36d5256f72ad4954d21f305cd8eff9a288 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 07:13:43 +0000 Subject: [PATCH 5/9] [rclone] improve harmony-rclone.sh Signed-off-by: Leo Chen --- scripts/package/harmony-rclone.sh | 67 +++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/scripts/package/harmony-rclone.sh b/scripts/package/harmony-rclone.sh index 1c24d55ff..f0602c30e 100755 --- a/scripts/package/harmony-rclone.sh +++ b/scripts/package/harmony-rclone.sh @@ -1,13 +1,64 @@ #!/bin/bash -if [ $# != 1 ]; then - echo "usage: $0 datadir" - exit 1 -fi +ME=$(basename "$0") + +function usage() { + local MSG=$1 + + cat<<-EOT +$MSG +This script will rclone the harmony db to datadir/archive directory. + +Usage: $ME [options] datadir shard + +datadir: the root directory of the harmony db (default: /home/harmony) +shard: the shard number to sync (valid value: 0,1,2,3) -HARMONY_HOME="$1" +Options: + -h print this help message + -c clean up backup db after rclone + -a sync archival db, instead of regular db -sudo -u harmony mkdir -p ${HARMONY_HOME}/archive -for i in 0 1 2 3; do - sudo -u harmony rclone sync -vvv hmy:pub.harmony.one/mainnet.min/harmony_db_$i ${HARMONY_HOME}/archive/harmony_db_$i > ${HARMONY_HOME}/archive/archive-$i.log 2>&1 +EOT + exit 1 +} + +CLEAN=false +FOLDER=mainnet.min + +while getopts ":hca" opt; do + case $opt in + c) CLEAN=true ;; + a) FOLDER=mainnet.archival ;; + *) usage ;; + esac done + +shift $((OPTIND - 1)) + +if [ $# != 2 ]; then + usage +fi + +DATADIR="$1" +SHARD="$2" + +if [ ! -d "$DATADIR" ]; then + usage "ERROR: no datadir directory: $DATADIR" +fi + +case "$SHARD" in + 0|1|2|3) ;; + *) usage "ERROR: invalid shard number: $SHARD" ;; +esac + +mkdir -p "${DATADIR}/archive" + +rclone sync -vvv "hmy:pub.harmony.one/${FOLDER}/harmony_db_${SHARD}" "${DATADIR}/archive/harmony_db_${SHARD}" > "${DATADIR}/archive/archive-${SHARD}.log" 2>&1 + +[ -d "${DATADIR}/harmony_db_${SHARD}" ] && mv -f "${DATADIR}/harmony_db_${SHARD}" "${DATADIR}/archive/harmony_db_${SHARD}.bak" +mv -f "${DATADIR}/archive/harmony_db_${SHARD}" "${DATADIR}/harmony_db_${SHARD}" + +if $CLEAN; then + rm -rf "${DATADIR}/archive/harmony_db_${SHARD}.bak" +fi From ca635bfedce1ba05d3aca0939441921a8216abe5 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 08:15:40 +0000 Subject: [PATCH 6/9] [rpm] improve rpm scripts [rpm] add change log [rpm] fix postun in spec file [rpm] make config files [rpm] add more changelog to test upgrade [rpm] add jq as dependency [rpm] right usage of %config Signed-off-by: Leo Chen --- scripts/package/deb/DEBIAN/postinst | 11 +++++------ scripts/package/rpm/harmony.spec | 25 +++++++++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/scripts/package/deb/DEBIAN/postinst b/scripts/package/deb/DEBIAN/postinst index 39e8221c5..e8cb79c02 100755 --- a/scripts/package/deb/DEBIAN/postinst +++ b/scripts/package/deb/DEBIAN/postinst @@ -1,20 +1,19 @@ #!/bin/sh -set -e - after_install() { - : + mkdir -p /home/harmony/.hmy/blskeys + mkdir -p /home/harmony/.config/rclone + chown -R harmony.harmony /home/harmony } after_upgrade() { : -# systemctl restart harmony } # Setup system users and groups -addgroup --quiet --system harmony -adduser --quiet --system --ingroup harmony --gecos "harmony validator account" harmony +getent group harmony >/dev/null || addgroup --quiet --system harmony +getent passwd harmony >/dev/null || adduser --quiet --system --ingroup harmony --gecos "harmony validator account" harmony if [ "${1}" = "configure" -a -z "${2}" ] || \ [ "${1}" = "abort-remove" ] diff --git a/scripts/package/rpm/harmony.spec b/scripts/package/rpm/harmony.spec index 9ff93c9b1..4cdde97d3 100644 --- a/scripts/package/rpm/harmony.spec +++ b/scripts/package/rpm/harmony.spec @@ -16,7 +16,7 @@ Source0: %{name}-%{version}.tar BuildArch: x86_64 Packager: Leo Chen Requires(pre): shadow-utils -Requires: systemd-rpm-macros +Requires: systemd-rpm-macros jq %description Harmony is a sharded, fast finality, low fee, PoS public blockchain. @@ -58,14 +58,17 @@ install -m 0644 harmony.conf ${RPM_BUILD_ROOT}/etc/harmony/ exit 0 %post -%systemd_post %{name}.service +%systemd_user_post %{name}.service %sysctl_apply %{name}-sysctl.conf +exit 0 %preun -%systemd_preun %{name}.service +%systemd_user_preun %{name}.service +exit 0 %postun -%systemd_postun_with_restart ${name}.service +%systemd_postun_with_restart %{name}.service +exit 0 %files /usr/sbin/harmony @@ -77,11 +80,21 @@ exit 0 /etc/harmony/rclone.conf /home/harmony/.config/rclone +%config(noreplace) /etc/harmony/harmony.conf +%config /etc/harmony/rclone.conf +%config /etc/sysctl.d/99-harmony.conf +%config /etc/systemd/system/harmony.service + %doc %license %changelog -* Tue Aug 18 2020 Leo Chen 2.3.5 - - init version of the harmony node program +* Wed Aug 26 2020 Leo Chen 2.3.5 + - get version from git tag + - add %config macro to keep edited config files + +* Tue Aug 18 2020 Leo Chen 2.3.4 + - init version of the harmony node program + From 419967d2743fc70832ce10562cb82bbcf9eb5097 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 09:28:06 +0000 Subject: [PATCH 7/9] [deb] can not remove local repo [deb] improve deb scripts [deb] restart harmony after upgrade Signed-off-by: Leo Chen --- scripts/package/deb/DEBIAN/postinst | 6 +++++- scripts/package/deb/DEBIAN/postrm | 8 +++++++- scripts/package/deb/DEBIAN/prerm | 2 +- scripts/package/deb/changelog | 1 + scripts/package/publish-repo.sh | 2 -- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/package/deb/DEBIAN/postinst b/scripts/package/deb/DEBIAN/postinst index e8cb79c02..3a05d1e9c 100755 --- a/scripts/package/deb/DEBIAN/postinst +++ b/scripts/package/deb/DEBIAN/postinst @@ -4,10 +4,14 @@ after_install() { mkdir -p /home/harmony/.hmy/blskeys mkdir -p /home/harmony/.config/rclone chown -R harmony.harmony /home/harmony + + systemctl enable harmony + systemctl start harmony } after_upgrade() { - : + systemctl enable harmony + systemctl restart harmony } diff --git a/scripts/package/deb/DEBIAN/postrm b/scripts/package/deb/DEBIAN/postrm index 1b88a21c8..b3fcc11ae 100755 --- a/scripts/package/deb/DEBIAN/postrm +++ b/scripts/package/deb/DEBIAN/postrm @@ -1,4 +1,7 @@ #!/bin/sh + +set -e + after_remove() { : #rm -rf /data/harmony/latest @@ -7,7 +10,10 @@ after_remove() { } after_purge() { - : + systemctl disable harmony + userdel harmony || true + groupdel harmony || true + rm -rf /etc/harmony } dummy() { diff --git a/scripts/package/deb/DEBIAN/prerm b/scripts/package/deb/DEBIAN/prerm index 6f41586b4..6c4c5c22c 100755 --- a/scripts/package/deb/DEBIAN/prerm +++ b/scripts/package/deb/DEBIAN/prerm @@ -3,7 +3,7 @@ set -e before_remove() { - : + systemctl stop harmony } dummy() { diff --git a/scripts/package/deb/changelog b/scripts/package/deb/changelog index 103f4cd56..ca7d2cd24 100644 --- a/scripts/package/deb/changelog +++ b/scripts/package/deb/changelog @@ -1 +1,2 @@ +1.1 improved debian scripts 1.0 init harmony debian package diff --git a/scripts/package/publish-repo.sh b/scripts/package/publish-repo.sh index 795894d3b..db4b538f9 100755 --- a/scripts/package/publish-repo.sh +++ b/scripts/package/publish-repo.sh @@ -77,8 +77,6 @@ function publish_deb() { aptly repo add harmony-$PROFILE $SRC aptly publish repo harmony-$PROFILE s3:harmony-$PROFILE: fi - # remove the local repo - aptly repo drop harmony-$PROFILE } ################## MAIN ################## From 03f6d23352b814f0c55bdad437c856e554396196 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 10:21:01 +0000 Subject: [PATCH 8/9] [deb] add recommended dependencies Signed-off-by: Leo Chen --- scripts/package/deb/DEBIAN/control | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/package/deb/DEBIAN/control b/scripts/package/deb/DEBIAN/control index 805e021b1..98861ff5d 100644 --- a/scripts/package/deb/DEBIAN/control +++ b/scripts/package/deb/DEBIAN/control @@ -11,3 +11,4 @@ Priority: extra Homepage: https://harmony.one Description: Harmony is a sharded, fast finality, low fee, PoS public blockchain. This package contains the validator node program for harmony blockchain. Build-Dep: dh-systemd +Recommends: jq, rclone (>= 1.52.3) From 2b64ac237bce264dbd3c4b8b23248acd4056cdb1 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Wed, 26 Aug 2020 10:24:03 +0000 Subject: [PATCH 9/9] [Makefile] fixed the release ver Signed-off-by: Leo Chen --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1b289e31b..612697de4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export DYLD_FALLBACK_LIBRARY_PATH:=$(LD_LIBRARY_PATH) export GO111MODULE:=on PKGNAME=harmony VERSION?=$(shell git tag -l --sort=-v:refname | head -n 1 | tr -d v) -RELEASE?=$(shell git describe | cut -f2 -d-) +RELEASE?=$(shell git describe --long | cut -f2 -d-) RPMBUILD=$(HOME)/rpmbuild DEBBUILD=$(HOME)/debbuild SHELL := bash