From 4c2786b799367ceed1cec6fb748417c2c9384c5d Mon Sep 17 00:00:00 2001 From: Favour Date: Sat, 19 Oct 2024 13:01:49 +0100 Subject: [PATCH] fix: improved efficiency of deduplication in verification.json script --- .../infra/scripts/dedupe-verification-json.sh | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/typescript/infra/scripts/dedupe-verification-json.sh b/typescript/infra/scripts/dedupe-verification-json.sh index 58cbb9ddd..1b732fa03 100755 --- a/typescript/infra/scripts/dedupe-verification-json.sh +++ b/typescript/infra/scripts/dedupe-verification-json.sh @@ -19,17 +19,33 @@ verification_files=$(find "$config_dir" -name "verification.json") for input_file in $verification_files; do echo "Processing file: $input_file" + # Check if the file contains valid JSON + if ! jq empty "$input_file" &>/dev/null; then + echo "Error: $input_file contains invalid JSON. Skipping file." + continue + fi + # Get the list of chains chains=$(jq -r 'keys[]' "$input_file") - # Iterate through each chain and deduplicate its array in-place + # Iterate through each chain and deduplicate its array for chain in $chains; do echo "Deduplicating array for chain: $chain" - jq ".$chain |= (reduce .[] as \$item ([]; if any(.[]; .address == \$item.address) then . else . + [\$item] end))" "$input_file" > temp.json && mv temp.json "$input_file" + + # Deduplicate in memory and only write back if changes are made + updated_json=$(jq ".$chain |= unique_by(.address)" "$input_file") + + # Only overwrite if there are changes + if [ "$updated_json" != "$(cat "$input_file")" ]; then + echo "$updated_json" > "$input_file" + echo "File has been updated and deduplicated in-place: $input_file" + else + echo "No changes needed for $input_file" + fi done - echo "File has been deduplicated in-place: $input_file" echo done echo "All verification.json files have been processed." +