[Explorer] Apply a delay to transaction fetching calls to Etherscan to avoid being rate limited

pull/6976/head
Hwee-Boon Yar 1 year ago
parent 1b648e24d4
commit 237d68e6f7
  1. 235
      modules/AlphaWalletFoundation/AlphaWalletFoundation/Tokens/Logic/EtherscanCompatibleBlockchainExplorer.swift

@ -33,23 +33,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
guard let pagination = (pagination ?? defaultPagination) as? BlockBasedPagination else {
return .fail(PromiseError(error: BlockchainExplorerError.paginationTypeNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: pagination.startBlock,
endBlock: pagination.endBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .tokentx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: pagination.startBlock, endBlock: pagination.endBlock, apiKey: apiKey, walletAddress: walletAddress, action: .tokentx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc20) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc20) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
func erc721TokenInteractions(walletAddress: AlphaWallet.Address,
@ -65,22 +61,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
return .fail(PromiseError(error: BlockchainExplorerError.methodNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: pagination.startBlock,
endBlock: pagination.endBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .tokennfttx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: pagination.startBlock, endBlock: pagination.endBlock, apiKey: apiKey, walletAddress: walletAddress, action: .tokennfttx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc721) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc721) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
func erc1155TokenInteractions(walletAddress: AlphaWallet.Address,
@ -96,22 +89,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
return .fail(PromiseError(error: BlockchainExplorerError.methodNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: pagination.startBlock,
endBlock: pagination.endBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .token1155tx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: pagination.startBlock, endBlock: pagination.endBlock, apiKey: apiKey, walletAddress: walletAddress, action: .token1155tx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc1155) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { UniqueNonEmptyContracts(json: try JSON(data: $0.data), tokenType: .erc1155) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
func erc20TokenTransferTransactions(walletAddress: AlphaWallet.Address,
@ -166,51 +156,44 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
return .fail(PromiseError(error: BlockchainExplorerError.paginationTypeNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: pagination.startBlock,
endBlock: pagination.endBlock,
sortOrder: sortOrder,
apiKey: apiKey,
walletAddress: walletAddress,
action: .txlist)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: pagination.startBlock, endBlock: pagination.endBlock, sortOrder: sortOrder, apiKey: apiKey, walletAddress: walletAddress, action: .txlist, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.mapError { PromiseError(error: $0) }
.flatMap { [transactionBuilder] result -> AnyPublisher<TransactionsResponse, PromiseError> in
if result.response.statusCode == 404 {
return .fail(.some(error: URLError(URLError.Code(rawValue: 404)))) // Clearer than a JSON deserialization error when it's a 404
}
do {
let promises = try JSONDecoder().decode(ArrayResponse<NormalTransaction>.self, from: result.data)
.result.map { transactionBuilder.buildTransaction(from: $0) }
return Publishers.MergeMany(promises)
.collect()
.map {
let transactions = functional.filter(
transactions: $0.compactMap { $0 },
startBlock: pagination.startBlock,
endBlock: pagination.endBlock)
let (_, _, maxBlockNumber) = functional.extractBoundingBlockNumbers(fromTransactions: transactions)
if maxBlockNumber > 0 {
let nextPage = BlockBasedPagination(startBlock: maxBlockNumber + 1, endBlock: nil)
return TransactionsResponse(transactions: transactions, nextPage: nextPage)
} else {
return TransactionsResponse(transactions: transactions, nextPage: nil)
}
}.setFailureType(to: PromiseError.self)
.eraseToAnyPublisher()
} catch {
return .fail(.some(error: error))
}
}.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.mapError { PromiseError(error: $0) }
.flatMap { [transactionBuilder] result -> AnyPublisher<TransactionsResponse, PromiseError> in
if result.response.statusCode == 404 {
return .fail(.some(error: URLError(URLError.Code(rawValue: 404)))) // Clearer than a JSON deserialization error when it's a 404
}
do {
let promises = try JSONDecoder().decode(ArrayResponse<NormalTransaction>.self, from: result.data)
.result.map { transactionBuilder.buildTransaction(from: $0) }
return Publishers.MergeMany(promises)
.collect()
.map {
let transactions = functional.filter(
transactions: $0.compactMap { $0 },
startBlock: pagination.startBlock,
endBlock: pagination.endBlock)
let (_, _, maxBlockNumber) = functional.extractBoundingBlockNumbers(fromTransactions: transactions)
if maxBlockNumber > 0 {
let nextPage = BlockBasedPagination(startBlock: maxBlockNumber + 1, endBlock: nil)
return TransactionsResponse(transactions: transactions, nextPage: nextPage)
} else {
return TransactionsResponse(transactions: transactions, nextPage: nil)
}
}.setFailureType(to: PromiseError.self)
.eraseToAnyPublisher()
} catch {
return .fail(.some(error: error))
}
}.eraseToAnyPublisher()
}
func erc1155TokenTransferTransaction(walletAddress: AlphaWallet.Address,
@ -245,21 +228,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
server: RPCServer,
startBlock: Int? = nil) -> AnyPublisher<[Transaction], PromiseError> {
let request = Request(
baseUrl: baseUrl,
startBlock: startBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .tokentx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: startBlock, apiKey: apiKey, walletAddress: walletAddress, action: .tokentx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc20) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc20) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
private func getErc721Transactions(walletAddress: AlphaWallet.Address,
@ -271,21 +252,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
return .fail(PromiseError(error: BlockchainExplorerError.methodNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: startBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .tokennfttx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: startBlock, apiKey: apiKey, walletAddress: walletAddress, action: .tokennfttx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc721) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc721) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
private func getErc1155Transactions(walletAddress: AlphaWallet.Address,
@ -297,21 +276,19 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
return .fail(PromiseError(error: BlockchainExplorerError.methodNotSupported))
}
let request = Request(
baseUrl: baseUrl,
startBlock: startBlock,
apiKey: apiKey,
walletAddress: walletAddress,
action: .token1155tx)
let delay = self.randomDelay()
let request = Request(baseUrl: baseUrl, startBlock: startBlock, apiKey: apiKey, walletAddress: walletAddress, action: .token1155tx, delay: delay)
let analytics = analytics
let domainName = baseUrl.host!
return transporter
.dataTaskPublisher(request)
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc1155) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
return Just(Void())
.delay(for: .seconds(delay), scheduler: DispatchQueue.main)
.setFailureType(to: SessionTaskError.self)
.flatMap { _ in self.transporter.dataTaskPublisher(request) }
.handleEvents(receiveOutput: { [server] in Self.log(response: $0, server: server, analytics: analytics, domainName: domainName) })
.tryMap { functional.decodeTokenTransferTransactions(json: JSON($0.data), server: server, tokenType: .erc1155) }
.mapError { PromiseError.some(error: $0) }
.eraseToAnyPublisher()
}
private func backFillTransactionGroup(walletAddress: AlphaWallet.Address,
@ -374,6 +351,11 @@ class EtherscanCompatibleBlockchainExplorer: BlockchainExplorer {
}
}
}
//For avoid being rate limited
private func randomDelay() -> Int {
Int.random(in: 4...30)
}
}
// swiftlint:enable type_body_length
@ -431,15 +413,10 @@ extension EtherscanCompatibleBlockchainExplorer {
let walletAddress: AlphaWallet.Address
let sortOrder: GetTransactions.SortOrder?
let action: Action
//This is just displayed as part of the URL for debugging
let delay: Int
init(baseUrl: URL,
startBlock: Int? = nil,
endBlock: Int? = nil,
sortOrder: GetTransactions.SortOrder? = nil,
apiKey: String? = nil,
walletAddress: AlphaWallet.Address,
action: Action) {
init(baseUrl: URL, startBlock: Int? = nil, endBlock: Int? = nil, sortOrder: GetTransactions.SortOrder? = nil, apiKey: String? = nil, walletAddress: AlphaWallet.Address, action: Action, delay: Int = 0) {
self.action = action
self.baseUrl = baseUrl
self.sortOrder = sortOrder
@ -447,6 +424,7 @@ extension EtherscanCompatibleBlockchainExplorer {
self.endBlock = endBlock
self.apiKey = apiKey
self.walletAddress = walletAddress
self.delay = delay
}
func asURLRequest() throws -> URLRequest {
@ -473,6 +451,11 @@ extension EtherscanCompatibleBlockchainExplorer {
params["sort"] = sortOrder.rawValue
}
if AlphaWallet.Device.isSimulator {
//Helpful for debugging rate limiting since we can see the delay applied in the URL itself
params["delay"] = delay
}
request.allHTTPHeaderFields = [
"Content-Type": "application/json",
"client": Bundle.main.bundleIdentifier ?? "",

Loading…
Cancel
Save