From f71e542f467cf9d9ab4d2c1edf7e760373c31b6d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 3 Apr 2018 10:56:18 -0500 Subject: [PATCH] Migrate from CircleCI 1.0 to 2.0 Note: deployment jobs are filtered by branch, so deploy_staging won't be tested until the merge to master and deploy_production won't be tested until the merge to production. --- .circleci/config.yml | 384 +++++++++++++++++++++++++ .circleci/elixir-node-ruby/Dockerfile | 83 ++++++ .circleci/elixir-node-ruby/README.md | 20 ++ .circleci/setup-heroku.sh | 16 ++ .credo.exs | 2 +- .tool-versions | 5 +- apps/explorer/test/test_helper.exs | 5 + apps/explorer_web/test/test_helper.exs | 5 + circle.yml | 78 ----- elixir_buildpack.config | 4 +- 10 files changed, 519 insertions(+), 83 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/elixir-node-ruby/Dockerfile create mode 100644 .circleci/elixir-node-ruby/README.md create mode 100644 .circleci/setup-heroku.sh delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..ee6dd51b21 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,384 @@ +version: 2 +jobs: + build: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4-node-browsers + environment: + MIX_ENV: test + # match POSTGRES_PASSWORD for postgres image below + PGPASSWORD: postgres + # match POSTGRES_USER for postgres image below + PGUSER: postgres + + working_directory: ~/app + + steps: + - checkout + + - run: mix local.hex --force + - run: mix local.rebar --force + + - restore_cache: + keys: + - v2-mix-deps-get-{{ checksum "mix.lock" }} + - v2-mix-deps-get-{{ checksum "mix.exs" }} + - v2-mix-deps-get + + - run: mix deps.get + + - save_cache: + key: v2-mix-deps-get-{{ checksum "mix.lock" }} + paths: "deps" + - save_cache: + key: mix-deps-get-{{ checksum "mix.exs" }} + paths: "deps" + - save_cache: + key: mix-deps-get + paths: "deps" + + - restore_cache: + keys: + - v2-npm-install-{{ .Branch }}-{{ checksum "apps/explorer_web/assets/package-lock.json" }} + - v2-npm-install-{{ .Branch }} + - v2-npm-install + + - run: + command: npm install + working_directory: "apps/explorer_web/assets" + + - save_cache: + key: v2-npm-install-{{ .Branch }}-{{ checksum "apps/explorer_web/assets/package-lock.json" }} + paths: "apps/explorer_web/assets/node_modules" + - save_cache: + key: v2-npm-install-{{ .Branch }} + paths: "apps/explorer_web/assets/node_modules" + - save_cache: + key: v2-npm-install + paths: "apps/explorer_web/assets/node_modules" + + - run: + name: "ELIXIR_VERSION.lock" + command: echo "${ELIXIR_VERSION}" > ELIXIR_VERSION.lock + - run: + name: "OTP_VERSION.lock" + command: echo "${OTP_VERSION}" > OTP_VERSION.lock + + - restore_cache: + keys: + - v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + - v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + - v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + + - run: mix compile + + - save_cache: + key: v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + paths: + - _build + - save_cache: + key: v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + paths: + - _build + - save_cache: + key: v2-mix-compile-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + paths: + - _build + + - run: + name: Build assets + command: node node_modules/brunch/bin/brunch build + working_directory: "apps/explorer_web/assets" + + - persist_to_workspace: + root: . + paths: + - _build + - apps + - config + - deps + - doc + - .credo.exs + - .dialyzer-ignore + - ELIXIR_VERSION.lock + - .formatter.exs + - .sobelow-conf + - Gemfile + - Gemfile.lock + - mix.exs + - mix.lock + - OTP_VERSION.lock + check_formatted: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix format --check-formatted + credo: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix local.hex --force + + - run: mix credo + deploy_production: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - add_ssh_keys: + fingerprints: + - "c4:fd:a8:f8:48:a8:09:e5:3e:be:30:62:4d:6f:6f:36" + + - run: + name: Setup Heroku + command: bash .circleci/setup-heroku.sh + + - run: + name: Deploy Production to Heroku + command: bin/deploy poa-explorer-production + deploy_staging: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - add_ssh_keys: + fingerprints: + - "c4:fd:a8:f8:48:a8:09:e5:3e:be:30:62:4d:6f:6f:36" + + - run: + name: Setup Heroku + command: bash .circleci/setup-heroku.sh + + - run: + name: Deploy Staging to Heroku + command: bin/deploy poa-explorer-staging + dialyzer: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix local.hex --force + + - restore_cache: + keys: + - v2-mix-dailyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + - v2-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + - v2-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + + - run: + name: Unpack PLT cache + command: | + mkdir -p _build/test + cp plts/dialyxir*.plt _build/test/ || true + mkdir -p ~/.mix + cp plts/dialyxir*.plt ~/.mix/ || true + + - run: mix dialyzer --plt + + - run: + name: Pack PLT cache + command: | + mkdir -p plts + cp _build/test/dialyxir*.plt plts/ + cp ~/.mix/dialyxir*.plt plts/ + + - save_cache: + key: v2-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.lock" }} + paths: + - plts + - save_cache: + key: v1-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }}-{{ checksum "mix.exs" }} + paths: + - plts + - save_cache: + key: v1-mix-dialyzer-{{ checksum "OTP_VERSION.lock" }}-{{ checksum "ELIXIR_VERSION.lock" }} + paths: + - plts + + - run: mix dialyzer + eslint: + docker: + # Ensure .tool-versions matches + - image: circleci/node:9.10.1 + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: + name: ESLint + command: ./node_modules/.bin/eslint --format=junit --output-file="test/eslint/junit.xml" js/**/*.js + working_directory: apps/explorer_web/assets + + - store_test_results: + path: apps/explorer_web/assets/test + license_finder: + docker: + # Ensure .tool-versions matches + - image: poanetwork/elixir:1.6.4-node-ruby + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix local.hex --force + + - run: + name: Bundle Install + command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 + - run: + name: License Finder Action Items + command: bundle exec license_finder --project-path=. --decisions-file=doc/dependency_decisions.yml + sobelow: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4 + environment: + MIX_ENV: test + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix local.hex --force + + - run: mix sobelow --config + test: + docker: + # Ensure .tool-versions matches + - image: circleci/elixir:1.6.4-node-browsers + environment: + MIX_ENV: test + # match POSTGRES_PASSWORD for postgres image below + PGPASSWORD: postgres + # match POSTGRES_USER for postgres image below + PGUSER: postgres + - image: circleci/postgres:10.3-alpine + environment: + # Match apps/explorer/config/test.exs config :explorerer, Explorer.Repo, database + POSTGRES_DB: explorer_test + # match PGPASSWORD for elixir image above + POSTGRES_PASSWORD: postgres + # match PGUSER for elixir image above + POSTGRES_USER: postgres + - image: circleci/redis:4.0.9-alpine + + working_directory: ~/app + + steps: + - attach_workspace: + at: . + + - run: mix local.hex --force + - run: mix local.rebar --force + + - run: + name: Wait for DB + command: dockerize -wait tcp://localhost:5432 -timeout 1m + + - run: + name: Wait for Redis + command: dockerize -wait tcp://localhost:6379 -timeout 1m + + - run: mix test + + - store_test_results: + path: _build/test/junit +workflows: + version: 2 + primary: + jobs: + - build + - check_formatted: + requires: + - build + - credo: + requires: + - build + - deploy_production: + filters: + branches: + only: production + requires: + - check_formatted + - credo + - eslint + - license_finder + - sobelow + - test + - deploy_staging: + filters: + branches: + only: master + requires: + - check_formatted + - credo + - eslint + - license_finder + - sobelow + - test + - dialyzer: + requires: + - build + - eslint: + requires: + - build + - license_finder: + requires: + - build + - sobelow: + requires: + - build + - test: + requires: + - build diff --git a/.circleci/elixir-node-ruby/Dockerfile b/.circleci/elixir-node-ruby/Dockerfile new file mode 100644 index 0000000000..7f23c6c182 --- /dev/null +++ b/.circleci/elixir-node-ruby/Dockerfile @@ -0,0 +1,83 @@ +# MUST match base image version used in .circleci/config.yml. +# -browser variant is dropped from name because license_finder doesn't need browsers +FROM circleci/elixir:1.6.4-node + +USER root + +# https://github.com/CircleCI-Public/circleci-dockerfiles/blob/1da7b6007c52a166741b6e15a423e7729408070d/ruby/images/2.5.1-stretch/Dockerfile uses ruby:2.5.1-stretch, so inline ruby:2.5.1-stretch: +# https://github.com/docker-library/ruby/blob/c9644fe5c95cd71913db348baa41240f05d882b3/2.5/stretch/Dockerfile + +# skip installing gem documentation +RUN mkdir -p /usr/local/etc \ + && { \ + echo 'install: --no-document'; \ + echo 'update: --no-document'; \ + } >> /usr/local/etc/gemrc + +ENV RUBY_MAJOR 2.5 +ENV RUBY_VERSION 2.5.1 +ENV RUBY_DOWNLOAD_SHA256 886ac5eed41e3b5fc699be837b0087a6a5a3d10f464087560d2d21b3e71b754d +ENV RUBYGEMS_VERSION 2.7.6 +ENV BUNDLER_VERSION 1.16.1 + +# some of ruby's build scripts are written in ruby +# we purge system ruby later to make sure our final image uses what we just built +RUN set -ex \ + \ + && buildDeps=' \ + bison \ + dpkg-dev \ + libgdbm-dev \ + ruby \ + ' \ + && apt-get update \ + && apt-get install -y --no-install-recommends $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + \ + && wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" \ + && echo "$RUBY_DOWNLOAD_SHA256 *ruby.tar.xz" | sha256sum -c - \ + \ + && mkdir -p /usr/src/ruby \ + && tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 \ + && rm ruby.tar.xz \ + \ + && cd /usr/src/ruby \ + \ +# hack in "ENABLE_PATH_CHECK" disabling to suppress: +# warning: Insecure world writable dir + && { \ + echo '#define ENABLE_PATH_CHECK 0'; \ + echo; \ + cat file.c; \ + } > file.c.new \ + && mv file.c.new file.c \ + \ + && autoconf \ + && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ + && ./configure \ + --build="$gnuArch" \ + --disable-install-doc \ + --enable-shared \ + && make -j "$(nproc)" \ + && make install \ + \ + && apt-get purge -y --auto-remove $buildDeps \ + && cd / \ + && rm -r /usr/src/ruby \ + \ + && gem update --system "$RUBYGEMS_VERSION" \ + && gem install bundler --version "$BUNDLER_VERSION" --force \ + && rm -r /root/.gem/ + +# install things globally, for great justice +# and don't create ".bundle" in all our apps +ENV GEM_HOME /usr/local/bundle +ENV BUNDLE_PATH="$GEM_HOME" \ + BUNDLE_BIN="$GEM_HOME/bin" \ + BUNDLE_SILENCE_ROOT_WARNING=1 \ + BUNDLE_APP_CONFIG="$GEM_HOME" +ENV PATH $BUNDLE_BIN:$PATH +RUN mkdir -p "$GEM_HOME" "$BUNDLE_BIN" \ + && chmod 777 "$GEM_HOME" "$BUNDLE_BIN" + +USER circleci diff --git a/.circleci/elixir-node-ruby/README.md b/.circleci/elixir-node-ruby/README.md new file mode 100644 index 0000000000..2733fdf1b9 --- /dev/null +++ b/.circleci/elixir-node-ruby/README.md @@ -0,0 +1,20 @@ +# elixir-node-ruby + +The `elixir-node-ruby` is a variant of `circleci/elixir` Dockerfiles that adds Ruby on top of `elixir:*-node`, so that `license_finder`, a Ruby Gem, can scan Elixir (`mix`) and Node dependencies in [POA Network](https://github.com/poanetwork) repositories, such as https://github.com/poanetwork/poa-explorer. + +## Building + +1. `cd .circleci/elixir-node-ruby` +2. `docker build -t poanetwork/elixir:1.6.4-node-ruby .` + +## Testing + +1. `docker run -it poanetwork/elixir:1.6.4-node-ruby /bin/bash` +2. Run IRB to check for ruby: `irb` +3. In IRB, quit: `quit` +4. Exit container shell: `exit` + +## Publishing + +1. Login to DockerHub from the Docker CLI: `docker login` +2. Push image `docker push poanetwork/elixir:1.6.4-node-ruby` diff --git a/.circleci/setup-heroku.sh b/.circleci/setup-heroku.sh new file mode 100644 index 0000000000..a771f2c3ec --- /dev/null +++ b/.circleci/setup-heroku.sh @@ -0,0 +1,16 @@ + #!/bin/bash + wget https://cli-assets.heroku.com/branches/stable/heroku-linux-amd64.tar.gz + sudo mkdir -p /usr/local/lib /usr/local/bin + sudo tar -xvzf heroku-linux-amd64.tar.gz -C /usr/local/lib + sudo ln -s /usr/local/lib/heroku/bin/heroku /usr/local/bin/heroku + + cat > ~/.netrc << EOF + machine api.heroku.com + login $HEROKU_LOGIN + password $HEROKU_API_KEY + EOF + + cat >> ~/.ssh/config << EOF + VerifyHostKeyDNS yes + StrictHostKeyChecking no + EOF diff --git a/.credo.exs b/.credo.exs index 29efa3dc50..f30210ff4d 100644 --- a/.credo.exs +++ b/.credo.exs @@ -33,7 +33,7 @@ # If you want to enforce a style guide and need a more traditional linting # experience, you can change `strict` to `true` below: # - strict: false, + strict: true, # # If you want to use uncolored output by default, you can change `color` # to `false` below: diff --git a/.tool-versions b/.tool-versions index dfb342c701..c49a241442 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,5 @@ -elixir 1.6.1 -erlang 20.2.4 +elixir 1.6.4 +erlang 20.3.2 +nodejs 9.10.1 ruby 2.4.1 nodejs 9.10.1 diff --git a/apps/explorer/test/test_helper.exs b/apps/explorer/test/test_helper.exs index 299a2852c6..a317e29531 100644 --- a/apps/explorer/test/test_helper.exs +++ b/apps/explorer/test/test_helper.exs @@ -1,3 +1,8 @@ +# https://github.com/CircleCI-Public/circleci-demo-elixir-phoenix/blob/a89de33a01df67b6773ac90adc74c34367a4a2d6/test/test_helper.exs#L1-L3 +junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config[:app]}" +File.mkdir_p!(junit_folder) +:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder) + ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) ExUnit.start() diff --git a/apps/explorer_web/test/test_helper.exs b/apps/explorer_web/test/test_helper.exs index 1e7133c111..cffded5eea 100644 --- a/apps/explorer_web/test/test_helper.exs +++ b/apps/explorer_web/test/test_helper.exs @@ -1,3 +1,8 @@ +# https://github.com/CircleCI-Public/circleci-demo-elixir-phoenix/blob/a89de33a01df67b6773ac90adc74c34367a4a2d6/test/test_helper.exs#L1-L3 +junit_folder = Mix.Project.build_path() <> "/junit/#{Mix.Project.config[:app]}" +File.mkdir_p!(junit_folder) +:ok = Application.put_env(:junit_formatter, :report_dir, junit_folder) + ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter]) ExUnit.start() diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 309383a746..0000000000 --- a/circle.yml +++ /dev/null @@ -1,78 +0,0 @@ -general: - artifacts: - - screenshots - -machine: - environment: - PATH: "$HOME/.asdf/bin:$HOME/.asdf/shims:$PATH" - MIX_ENV: "test" - node: - version: 9.4.0 - services: - - postgresql - - redis - pre: - - mkdir -p $CIRCLE_TEST_REPORTS/exunit - - mkdir -p $CIRCLE_TEST_REPORTS/eslint - -dependencies: - pre: - - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - - sudo dpkg -i google-chrome-stable_current_amd64.deb - - sudo sed -i 's|HERE/chrome\"|HERE/chrome\" --disable-setuid-sandbox|g' /opt/google/chrome/google-chrome - - rm google-chrome-stable_current_amd64.deb - - "LATEST_RELEASE=`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE` && wget https://chromedriver.storage.googleapis.com/${LATEST_RELEASE}/chromedriver_linux64.zip" - - unzip chromedriver_linux64.zip - - sudo cp chromedriver /usr/local/bin/chromedriver - - sudo chmod +x /usr/local/bin/chromedriver - - if ! asdf | grep version; then git clone https://github.com/HashNuke/asdf.git ~/.asdf; fi - - if ! asdf plugin-list | grep erlang; then asdf plugin-add erlang https://github.com/HashNuke/asdf-erlang.git; fi - - if ! asdf plugin-list | grep elixir; then asdf plugin-add elixir https://github.com/HashNuke/asdf-elixir.git; fi - - if ! asdf plugin-list | grep ruby; then asdf plugin-add ruby https://github.com/HashNuke/asdf-ruby.git; fi - - awk '/erlang/ { print $2 }' .tool-versions | xargs asdf install erlang: - timeout: 3600 - - awk '/elixir/ { print $2 }' .tool-versions | xargs asdf install elixir: - timeout: 3600 - - awk '/ruby/ { print $2 }' .tool-versions | xargs asdf install ruby: - timeout: 3600 - - gem install bundler - - mix local.hex --force: - timeout: 3600 - - mix local.rebar --force: - timeout: 3600 - override: - - mix do deps.get, deps.compile, compile - - bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 - - mix dialyzer --plt: - timeout: 3600 - - cd apps/explorer_web/assets && yarn install && yarn build && cd - - cache_directories: - - ~/.asdf - - _build - - deps - - apps/explorer_web/assets/node_modules - - vendor/bundle - -test: - pre: - - mix format --check-formatted - - mix credo - - mix sobelow --config - - mix dialyzer --halt-exit-status - - bundle exec license_finder --project-path=. --decisions-file=doc/dependency_decisions.yml - - cd apps/explorer_web/assets && bundle exec license_finder --decisions-file=../../..//doc/dependency_decisions.yml && cd - - - cd apps/explorer_web/assets && yarn eslint --format=junit --output-file="$CIRCLE_TEST_REPORTS/eslint/junit.xml" && cd - - override: - - mix test - post: - - cp _build/test/lib/explorer/test-junit-report.xml $CIRCLE_TEST_REPORTS/exunit - -deployment: - staging: - branch: master - commands: - - bin/deploy poa-explorer-staging - production: - branch: production - commands: - - bin/deploy poa-explorer-production diff --git a/elixir_buildpack.config b/elixir_buildpack.config index 597899b969..14addafdfb 100644 --- a/elixir_buildpack.config +++ b/elixir_buildpack.config @@ -1,3 +1,3 @@ -erlang_version=20.2 -elixir_version=1.6.1 +erlang_version=20.3.2 +elixir_version=1.6.4 always_rebuild=true