Merge pull request #1881 from poanetwork/store-solc-versions

fix: store solc versions locally for performance

CHANGELOG.md

@@ -22,6 +22,7 @@
 - [#1868](https://github.com/poanetwork/blockscout/pull/1868) - fix: logs list endpoint performance
 - [#1822](https://github.com/poanetwork/blockscout/pull/1822) - Fix style breaks in decompiled contract code view
 - [#1896](https://github.com/poanetwork/blockscout/pull/1896) - re-query tokens in top nav automplete
+- [#1881](https://github.com/poanetwork/blockscout/pull/1881) - fix: store solc versions locally for performance
 
 ### Chore
 

apps/explorer/.gitignore

@@ -1 +1,2 @@
-priv/.recovery
+priv/.recovery
+priv/solc_compilers/

apps/explorer/lib/explorer/application.ex

@@ -26,6 +26,7 @@ defmodule Explorer.Application do
       Supervisor.Spec.worker(SpandexDatadog.ApiServer, [datadog_opts()]),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, id: Explorer.MarketTaskSupervisor),
       Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor),
+      Explorer.SmartContract.SolcDownloader,
       {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents},
       {Admin.Recovery, [[], [name: Admin.Recovery]]},
       {TransactionCountCache, [[], []]}

apps/explorer/lib/explorer/smart_contract/solc_downloader.ex

@@ -0,0 +1,92 @@
+defmodule Explorer.SmartContract.SolcDownloader do
+  @moduledoc """
+  Checks to see if the requested solc compiler version exists, and if not it
+  downloads and stores the file.
+  """
+  use GenServer
+
+  alias Explorer.SmartContract.Solidity.CompilerVersion
+
+  @latest_compiler_refetch_time :timer.minutes(30)
+
+  def ensure_exists(version) do
+    path = file_path(version)
+
+    if File.exists?(path) do
+      path
+    else
+      {:ok, compiler_versions} = CompilerVersion.fetch_versions()
+
+      if version in compiler_versions do
+        GenServer.call(__MODULE__, {:ensure_exists, version}, 60_000)
+      else
+        false
+      end
+    end
+  end
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  # sobelow_skip ["Traversal"]
+  @impl true
+  def init([]) do
+    File.mkdir(compiler_dir())
+
+    {:ok, []}
+  end
+
+  # sobelow_skip ["Traversal"]
+  @impl true
+  def handle_call({:ensure_exists, version}, _from, state) do
+    path = file_path(version)
+
+    if fetch?(version, path) do
+      temp_path = file_path("#{version}-tmp")
+
+      contents = download(version)
+
+      file = File.open!(temp_path, [:write, :exclusive])
+
+      IO.binwrite(file, contents)
+
+      File.rename(temp_path, path)
+    end
+
+    {:reply, path, state}
+  end
+
+  defp fetch?("latest", path) do
+    case File.stat(path) do
+      {:error, :enoent} ->
+        true
+
+      {:ok, %{mtime: mtime}} ->
+        last_modified = NaiveDateTime.from_erl!(mtime)
+        diff = Timex.diff(NaiveDateTime.utc_now(), last_modified, :milliseconds)
+
+        diff > @latest_compiler_refetch_time
+    end
+  end
+
+  defp fetch?(_, path) do
+    not File.exists?(path)
+  end
+
+  defp file_path(version) do
+    Path.join(compiler_dir(), "#{version}.js")
+  end
+
+  defp compiler_dir do
+    Application.app_dir(:explorer, "priv/solc_compilers/")
+  end
+
+  defp download(version) do
+    download_path = "https://ethereum.github.io/solc-bin/bin/soljson-#{version}.js"
+
+    download_path
+    |> HTTPoison.get!([], timeout: 60_000)
+    |> Map.get(:body)
+  end
+end

apps/explorer/lib/explorer/smart_contract/solidity/code_compiler.ex

@@ -3,6 +3,8 @@ defmodule Explorer.SmartContract.Solidity.CodeCompiler do
   Module responsible to compile the Solidity code of a given Smart Contract.
   """
 
+  alias Explorer.SmartContract.SolcDownloader
+
   @new_contract_name "New.sol"
   @allowed_evm_versions ["homestead", "tangerineWhistle", "spuriousDragon", "byzantium", "constantinople", "petersburg"]
 
@@ -79,31 +81,36 @@ defmodule Explorer.SmartContract.Solidity.CodeCompiler do
         "byzantium"
       end
 
-    {response, _status} =
-      System.cmd(
-        "node",
-        [
-          Application.app_dir(:explorer, "priv/compile_solc.js"),
-          code,
-          compiler_version,
-          optimize_value(optimize),
-          optimization_runs,
-          @new_contract_name,
-          external_libs_string,
-          checked_evm_version
-        ]
-      )
-
-    with {:ok, contracts} <- Jason.decode(response),
-         %{"abi" => abi, "evm" => %{"deployedBytecode" => %{"object" => bytecode}}} <-
-           get_contract_info(contracts, name) do
-      {:ok, %{"abi" => abi, "bytecode" => bytecode, "name" => name}}
-    else
-      {:error, %Jason.DecodeError{}} ->
-        {:error, :compilation}
-
-      error ->
-        parse_error(error)
+    path = SolcDownloader.ensure_exists(compiler_version)
+
+    if path do
+      {response, _status} =
+        System.cmd(
+          "node",
+          [
+            Application.app_dir(:explorer, "priv/compile_solc.js"),
+            code,
+            compiler_version,
+            optimize_value(optimize),
+            optimization_runs,
+            @new_contract_name,
+            external_libs_string,
+            checked_evm_version,
+            path
+          ]
+        )
+
+      with {:ok, contracts} <- Jason.decode(response),
+           %{"abi" => abi, "evm" => %{"deployedBytecode" => %{"object" => bytecode}}} <-
+             get_contract_info(contracts, name) do
+        {:ok, %{"abi" => abi, "bytecode" => bytecode, "name" => name}}
+      else
+        {:error, %Jason.DecodeError{}} ->
+          {:error, :compilation}
+
+        error ->
+          parse_error(error)
+      end
     end
   end
 

apps/explorer/priv/compile_solc.js

@@ -1,7 +1,5 @@
 #!/usr/bin/env node
 
-const solc = require('solc');
-
 var sourceCode = process.argv[2];
 var version = process.argv[3];
 var optimize = process.argv[4];
@@ -9,38 +7,38 @@ var optimizationRuns = parseInt(process.argv[5], 10);
 var newContractName = process.argv[6];
 var externalLibraries = JSON.parse(process.argv[7])
 var evmVersion = process.argv[8];
+var compilerVersionPath = process.argv[9];
+
+var solc = require('solc')
+var compilerSnapshot = require(compilerVersionPath);
+var solc = solc.setupMethods(compilerSnapshot);
 
-var compiled_code = solc.loadRemoteVersion(version, function (err, solcSnapshot) {
-  if (err) {
-    console.log(JSON.stringify(err.message));
-  } else {
-    const input = {
-      language: 'Solidity',
-      sources: {
-        [newContractName]: {
-          content: sourceCode
-        }
-      },
-      settings: {
-        evmVersion: evmVersion,
-        optimizer: {
-          enabled: optimize == '1',
-          runs: optimizationRuns
-        },
-        libraries: {
-          [newContractName]: externalLibraries
-        },
-        outputSelection: {
-          '*': {
-            '*': ['*']
-          }
-        }
+const input = {
+  language: 'Solidity',
+  sources: {
+    [newContractName]: {
+      content: sourceCode
+    }
+  },
+  settings: {
+    evmVersion: evmVersion,
+    optimizer: {
+      enabled: optimize == '1',
+      runs: optimizationRuns
+    },
+    libraries: {
+      [newContractName]: externalLibraries
+    },
+    outputSelection: {
+      '*': {
+        '*': ['*']
       }
     }
-
-    const output = JSON.parse(solcSnapshot.compile(JSON.stringify(input)))
-    /** Older solc-bin versions don't use filename as contract key */
-    const response = output.contracts[newContractName] || output.contracts['']
-    console.log(JSON.stringify(response));
   }
-});
+}
+
+
+const output = JSON.parse(solc.compile(JSON.stringify(input)))
+/** Older solc-bin versions don't use filename as contract key */
+const response = output.contracts[newContractName] || output.contracts['']
+console.log(JSON.stringify(response));