updated CI and customised checks (#165)

- updated CI config to remove useless DIND -> now directly using the target
image with the correct soft installed as it was otherwise not possible to run
the jobs locally with Circle CI CLI.
- Updated and customised Vale config rules
- Updated en customised markdownlint config and rules
- moved config files to CI folder
- added scripts to run CI jobs locally with Circle CI CLI.

Signed-off-by: Nicolas MASSART <nicolas.massart@consensys.net>

.circleci/config.yml

@@ -10,6 +10,10 @@ executors:
   shell_executor:
     docker:
       - image: circleci/buildpack-deps:buster-scm
+  ruby_executor:
+    docker:
+      - image: circleci/ruby:2-stretch
+
 jobs:
   dco:
     executor: shell_executor
@@ -61,18 +65,17 @@ jobs:
     executor: python_executor
     steps:
       - checkout
-      - setup_remote_docker
+      - run:
+          name: Install dependencies
+          command: |
+            curl -sfL https://install.goreleaser.com/github.com/ValeLint/vale.sh | sh -s v1.7.1
       - run:
           name: Run Vale
           command: |
-            # create a dummy container which will hold a volume with data
-            docker create -v /doc --name docVol alpine:3.4 /bin/true
-            # copy files into this volume
-            docker cp ./ docVol:/doc
-            docker run --rm --volumes-from docVol jdkato/vale --config /doc/.vale.ini --glob='*.{md}' . > vale.out
+            ./bin/vale --config ./CI/vale_styles/.vale.ini --glob='*.{md}' . | tee ./vale.out
       - store_artifacts:
           path: ./vale.out
-          destination: vale.out
+          destination: ./vale.out
 
   linkchecker:
     executor: node_executor
@@ -101,21 +104,20 @@ jobs:
           destination: linkchecker.out
 
   markdownlint:
-    executor: python_executor
+    executor: ruby_executor
     steps:
       - checkout
-      - setup_remote_docker
+      - run:
+          name: Install dependencies
+          command: |
+            gem install mdl
       - run:
           name: Run Markdownlint
           command: |
-            # create a dummy container which will hold a volume with data
-            docker create -v /doc --name docVol alpine:3.4 /bin/true
-            # copy files into this volume
-            docker cp ./ docVol:/doc
-            docker run --rm --volumes-from docVol -w /doc pipelinecomponents/markdownlint mdl --style all . > markdownlint.out
+            mdl --ignore-front-matter --style ./CI/markdownlint/style.rb . | tee ./markdownlint.out
       - store_artifacts:
           path: ./markdownlint.out
-          destination: markdownlint.out
+          destination: ./markdownlint.out
 
 workflows:
   version: 2
@@ -126,3 +128,13 @@ workflows:
       - vale
       - markdownlint
       - linkchecker
+  nightly:
+    triggers:
+      - schedule: # GMT
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - linkchecker

.mdlrc

@@ -1,2 +0,0 @@
-rules "~MD034"
-show_kramdown_warnings true

CI/markdownlint/style.rb

@@ -0,0 +1,8 @@
+all
+
+rule "no-duplicate-header", :allow_different_nesting => true
+rule 'line-length', :line_length=>100, :code_blocks=>false, :tables=> false
+rule 'no-trailing-punctuation', :punctuation=>'.,;:!'
+
+exclude_rule 'no-bare-urls'
+exclude_rule 'code-block-style'

CI/scripts/README.md

@@ -0,0 +1,33 @@
+# Doc quality testing scripts
+
+Scripts in this directory can be used to run CircleCI jobs on your local machine before pushing your
+work to the Github repos.
+
+## Requirements
+
+* Install the [Circle CI local CLI](https://circleci.com/docs/2.0/local-cli/).
+* Install [Docker](https://docs.docker.com/install/)
+
+## Running the scripts
+
+Go to the besu-doc project root directory and run one of the following scripts:
+
+* `CI/scripts/test_build.sh` will run the doc build with MkDocs
+* `CI/scripts/test_guidelines.sh` will test the doc with Vale and our custom rules.
+* `CI/scripts/test_links.sh` will test links in the doc. Internal and external links are checked. If a link is incorrect or the targeted web page is unavailable (for external sites), the test will fail and display the faulty link.
+* `CI/scripts/test_lint.sh` will test the Markdown syntax for issues. Sometimes they are not visible but making sure the markdown is correct helps to make it readable and bug free.
+* `CI/scripts/test_all.sh` will test run all the test in one pass.
+
+# Known issues
+
+You will notice messages like:
+
+```
+====>> Saving Cache
+Error: 
+Skipping cache - error checking storage: not supported
+
+Step failed
+```
+
+This is normal, Circle CI doesn't support some of the features the server version does. Ignore them.

CI/scripts/test_all.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+CI/scripts/test_build.sh
+CI/scripts/test_guidelines.sh
+CI/scripts/test_links.sh
+CI/scripts/test_lint.sh

CI/scripts/test_build.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job build

CI/scripts/test_guidelines.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job vale

CI/scripts/test_links.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job linkchecker

CI/scripts/test_lint.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job markdownlint

CI/vale_styles/.vale.ini

@@ -1,12 +1,12 @@
 # Example Vale config file (`.vale.ini` or `_vale.ini`)
 
 # Core settings
-StylesPath = CI/vale_styles/
+StylesPath = vale_styles
 
 # The minimum alert level to display (suggestion, warning, or error).
 #
 # CI builds will only fail on error-level alerts.
-MinAlertLevel = suggestion
+MinAlertLevel = warning
 
 # The "formats" section allows you to associate an "unknown" format
 # with one of Vale's supported formats.
@@ -16,7 +16,7 @@ mdx = md
 # Global settings (applied to every syntax)
 [*]
 # List of styles to load
-BasedOnStyles = proselint, write-good, Joblint, Microsoft, Custom
+BasedOnStyles = proselint, write-good, Joblint, Microsoft, Besu
 # Style.Rule = {YES, NO} to enable or disable a specific rule
 vale.Editorializing = YES
 vale.Redundancy = YES
@@ -28,4 +28,13 @@ Microsoft.GeneralURL = NO
 Microsoft.Acronyms = NO
 Microsoft.Quotes = suggestion
 Microsoft.We = suggestion
-Microsoft.HeadingPunctuation = suggestion
+Microsoft.HeadingPunctuation = suggestion
+Microsoft.Headings = NO
+
+write-good.Weasel = NO
+
+proselint.Hyperbole = warning
+
+[common/CLA.md]
+# Disable guidelines check for this file as it's legal jargon
+BasedOnStyles =

package.json

@@ -7,9 +7,9 @@
     "doc": "docs"
   },
   "scripts": {
-    "test:links:check": "find . -name \\*.md ! -path \"./node_modules/*\" -exec ./node_modules/markdown-link-check/markdown-link-check -q -c link_check_conf.json {} \\; > linkchecker.out 2>&1",
-    "test:links:display": "cat ./linkchecker.out",
-    "test:links:verify": "! grep 'ERROR:' ./linkchecker.out",
+    "test:links:check": "find . -name \\*.md ! -path \"./node_modules/*\" -exec ./node_modules/markdown-link-check/markdown-link-check -q -c CI/linkchecker/link_check_conf.json {} \\; > linkchecker.out 2>&1",
+    "test:links:display": "cat linkchecker.out",
+    "test:links:verify": "! grep 'ERROR:' linkchecker.out",
     "test:links": "npm run test:links:check && npm run test:links:display && npm run test:links:verify",
     "test": "npm run test:links"
   },