Skip to content

Commit bdb02b0

Browse files
code-ashercode-asher
committed
Refactor retry
1 parent 1a67157 commit bdb02b0

File tree

4 files changed

+108
-91
lines changed

4 files changed

+108
-91
lines changed

‎src/main/kotlin/com/coder/gateway/CoderGatewayConnectionProvider.kt

Lines changed: 38 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package com.coder.gateway
44

55
import com.coder.gateway.sdk.humanizeDuration
6+
import com.coder.gateway.sdk.isCancellation
67
import com.coder.gateway.sdk.isWorkerTimeout
78
import com.coder.gateway.sdk.suspendingRetryWithExponentialBackOff
89
import com.coder.gateway.services.CoderRecentWorkspaceConnectionsService
@@ -33,44 +34,53 @@ class CoderGatewayConnectionProvider : GatewayConnectionProvider {
3334
// TODO: If this fails determine if it is an auth error and if so prompt
3435
// for a new token, configure the CLI, then try again.
3536
clientLifetime.launchUnderBackgroundProgress(CoderGatewayBundle.message("gateway.connector.coder.connection.provider.title"), canBeCancelled = true, isIndeterminate = true, project = null) {
36-
val context = suspendingRetryWithExponentialBackOff(
37-
label = "connect",
38-
logger = logger,
39-
action = { attempt ->
40-
logger.info("Deploying (attempt $attempt)...")
41-
indicator.text =
42-
if (attempt > 1) CoderGatewayBundle.message("gateway.connector.coder.connection.retry.text", attempt)
43-
else CoderGatewayBundle.message("gateway.connector.coder.connection.loading.text")
44-
SshMultistagePanelContext(parameters.toHostDeployInputs())
45-
},
46-
predicate = { e ->
47-
e is ConnectionException || e is TimeoutException
48-
|| e is SSHException || e is DeployException
49-
},
50-
update = { _, e, remainingMs ->
51-
if (remainingMs != null) {
37+
try {
38+
indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting")
39+
val context = suspendingRetryWithExponentialBackOff(
40+
action = { attempt ->
41+
logger.info("Connecting... (attempt $attempt")
42+
if (attempt > 1) {
43+
// indicator.text is the text above the progress bar.
44+
indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting.retry", attempt)
45+
}
46+
SshMultistagePanelContext(parameters.toHostDeployInputs())
47+
},
48+
retryIf = {
49+
it is ConnectionException || it is TimeoutException
50+
|| it is SSHException || it is DeployException
51+
},
52+
onError = { attempt, nextMs, e ->
53+
logger.error("Failed to connect (attempt $attempt; will retry in $nextMs ms)")
54+
// indicator.text2 is the text below the progress bar.
5255
indicator.text2 =
5356
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out"
5457
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
55-
indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connection.retry-error.text", humanizeDuration(remainingMs))
56-
} else {
57-
ApplicationManager.getApplication().invokeAndWait {
58-
Messages.showMessageDialog(
59-
e.message ?: CoderGatewayBundle.message("gateway.connector.no-details"),
60-
CoderGatewayBundle.message("gateway.connector.coder.connection.error.text"),
61-
Messages.getErrorIcon())
62-
}
63-
}
64-
},
65-
)
66-
if (context != null) {
58+
},
59+
onCountdown = { remainingMs ->
60+
indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting.failed.retry", humanizeDuration(remainingMs))
61+
},
62+
)
6763
launch {
6864
logger.info("Deploying and starting IDE with $context")
6965
// At this point JetBrains takes over with their own UI.
7066
@Suppress("UnstableApiUsage") SshDeployFlowUtil.fullDeployCycle(
7167
clientLifetime, context, Duration.ofMinutes(10)
7268
)
7369
}
70+
} catch (e: Exception) {
71+
if (isCancellation(e)) {
72+
logger.info("Connection canceled due to ${e.javaClass}")
73+
} else {
74+
logger.info("Failed to connect (will not retry)", e)
75+
// The dialog will close once we return so write the error
76+
// out into a new dialog.
77+
ApplicationManager.getApplication().invokeAndWait {
78+
Messages.showMessageDialog(
79+
e.message ?: CoderGatewayBundle.message("gateway.connector.no-details"),
80+
CoderGatewayBundle.message("gateway.connector.coder.connection.failed"),
81+
Messages.getErrorIcon())
82+
}
83+
}
7484
}
7585
}
7686

‎src/main/kotlin/com/coder/gateway/sdk/Retry.kt

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.coder.gateway.sdk
22

3-
import com.intellij.openapi.diagnostic.Logger
43
import com.intellij.openapi.progress.ProcessCanceledException
54
import com.intellij.ssh.SshException
65
import com.jetbrains.gateway.ssh.deploy.DeployException
@@ -22,25 +21,27 @@ fun unwrap(ex: Exception): Throwable {
2221
* Similar to Intellij's except it gives you the next delay, logs differently,
2322
* updates periodically (for counting down), runs forever, takes a predicate for
2423
* determining whether we should retry, and has some special handling for
25-
* exceptions to provide the true cause or better messages.
24+
* exceptions to provide the true cause.
2625
*
2726
* The update will have a boolean to indicate whether it is the first update (so
2827
* things like duplicate logs can be avoided). If remaining is null then no
2928
* more retries will be attempted.
3029
*
31-
* If an exception related to canceling is received then return null.
30+
* If an exception that cannot be retried is received (including those related
31+
* to canceling) that exception (after being unwrapped) will be thrown.
32+
*
33+
* onError will only be called on retryable errors.
3234
*/
3335
suspend fun <T> suspendingRetryWithExponentialBackOff(
3436
initialDelayMs: Long = TimeUnit.SECONDS.toMillis(5),
3537
backOffLimitMs: Long = TimeUnit.MINUTES.toMillis(3),
3638
backOffFactor: Int = 2,
3739
backOffJitter: Double = 0.1,
38-
label: String,
39-
logger: Logger,
40-
predicate: (e: Throwable) -> Boolean,
41-
update: (attempt: Int, e: Throwable, remaining: Long?) -> Unit,
42-
action: suspend (attempt: Int) -> T?
43-
): T? {
40+
retryIf: (e: Throwable) -> Boolean,
41+
onError: (attempt: Int, nextMs: Long, e: Throwable) -> Unit,
42+
onCountdown: (remaining: Long) -> Unit,
43+
action: suspend (attempt: Int) -> T
44+
): T {
4445
val random = Random()
4546
var delayMs = initialDelayMs
4647
for (attempt in 1..Int.MAX_VALUE) {
@@ -51,23 +52,13 @@ suspend fun <T> suspendingRetryWithExponentialBackOff(
5152
// SshException can happen due to anything from a timeout to being
5253
// canceled so unwrap to find out.
5354
val unwrappedEx = if (originalEx is SshException) unwrap(originalEx) else originalEx
54-
when (unwrappedEx) {
55-
is InterruptedException,
56-
is CancellationException,
57-
is ProcessCanceledException -> {
58-
logger.info("Retrying $label canceled due to ${unwrappedEx.javaClass}")
59-
return null
60-
}
61-
}
62-
if (!predicate(unwrappedEx)) {
63-
logger.error("Failed to $label (attempt $attempt; will not retry)", originalEx)
64-
update(attempt, unwrappedEx, null)
65-
return null
55+
if (!retryIf(unwrappedEx)) {
56+
throw unwrappedEx
6657
}
67-
logger.error("Failed to $label (attempt $attempt; will retry in $delayMs ms)", originalEx)
58+
onError(attempt, delayMs, unwrappedEx)
6859
var remainingMs = delayMs
6960
while (remainingMs > 0) {
70-
update(attempt, unwrappedEx, remainingMs)
61+
onCountdown(remainingMs)
7162
val next = min(remainingMs, TimeUnit.SECONDS.toMillis(1))
7263
remainingMs -= next
7364
delay(next)
@@ -98,3 +89,12 @@ fun humanizeDuration(durationMs: Long): String {
9889
fun isWorkerTimeout(e: Throwable): Boolean {
9990
return e is DeployException && e.message.contains("Worker binary deploy failed")
10091
}
92+
93+
/**
94+
* Return true if the exception is some kind of cancellation.
95+
*/
96+
fun isCancellation(e: Throwable): Boolean {
97+
return e is InterruptedException
98+
|| e is CancellationException
99+
|| e is ProcessCanceledException
100+
}

‎src/main/kotlin/com/coder/gateway/views/steps/CoderLocateRemoteProjectStepView.kt

Lines changed: 39 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import com.coder.gateway.sdk.CoderCLIManager
99
import com.coder.gateway.sdk.CoderRestClientService
1010
import com.coder.gateway.sdk.OS
1111
import com.coder.gateway.sdk.humanizeDuration
12+
import com.coder.gateway.sdk.isCancellation
1213
import com.coder.gateway.sdk.isWorkerTimeout
1314
import com.coder.gateway.sdk.suspendingRetryWithExponentialBackOff
1415
import com.coder.gateway.sdk.toURL
@@ -162,6 +163,7 @@ class CoderLocateRemoteProjectStepView(private val setNextButtonEnabled: (Boolea
162163
// Clear contents from the last attempt if any.
163164
cbIDEComment.foreground = UIUtil.getContextHelpForeground()
164165
cbIDEComment.text = CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.ide.none.comment")
166+
cbIDE.renderer = IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.retrieve-ides"))
165167
ideComboBoxModel.removeAllElements()
166168
setNextButtonEnabled(false)
167169

@@ -178,42 +180,47 @@ class CoderLocateRemoteProjectStepView(private val setNextButtonEnabled: (Boolea
178180
terminalLink.url = coderClient.coderURL.withPath("/@${coderClient.me.username}/${selectedWorkspace.name}/terminal").toString()
179181

180182
ideResolvingJob = cs.launch {
181-
val ides = suspendingRetryWithExponentialBackOff(
182-
label = "retrieve IDEs",
183-
logger = logger,
184-
action={ attempt ->
185-
logger.info("Deploying to ${selectedWorkspace.name} on $deploymentURL (attempt $attempt)")
186-
// Reset text in the select dropdown.
187-
withContext(Dispatchers.Main) {
188-
cbIDE.renderer = IDECellRenderer(
189-
if (attempt > 1) CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry.text", attempt)
190-
else CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.loading.text"))
191-
}
192-
val executor = createRemoteExecutor(CoderCLIManager.getHostName(deploymentURL, selectedWorkspace))
193-
if (ComponentValidator.getInstance(tfProject).isEmpty) {
194-
installRemotePathValidator(executor)
195-
}
196-
retrieveIDEs(executor, selectedWorkspace)
197-
},
198-
predicate = { e ->
199-
e is ConnectionException || e is TimeoutException
200-
|| e is SSHException || e is DeployException
201-
},
202-
update = { _, e, remainingMs ->
203-
cbIDEComment.foreground = UIUtil.getErrorForeground()
204-
cbIDEComment.text =
205-
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out. Check the command log for more details."
206-
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
207-
cbIDE.renderer =
208-
if (remainingMs != null) IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry-error.text", humanizeDuration(remainingMs)))
209-
else IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.error.text"), UIUtil.getBalloonErrorIcon())
210-
},
211-
)
212-
if (ides != null) {
183+
try {
184+
val ides = suspendingRetryWithExponentialBackOff(
185+
action = { attempt ->
186+
logger.info("Retrieving IDEs...(attempt $attempt)")
187+
if (attempt > 1) {
188+
cbIDE.renderer = IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.retrieve.ides.retry", attempt))
189+
}
190+
val executor = createRemoteExecutor(CoderCLIManager.getHostName(deploymentURL, selectedWorkspace))
191+
if (ComponentValidator.getInstance(tfProject).isEmpty) {
192+
installRemotePathValidator(executor)
193+
}
194+
retrieveIDEs(executor, selectedWorkspace)
195+
},
196+
retryIf = {
197+
it is ConnectionException || it is TimeoutException
198+
|| it is SSHException || it is DeployException
199+
},
200+
onError = { attempt, nextMs, e ->
201+
logger.error("Failed to retrieve IDEs (attempt $attempt; will retry in $nextMs ms)")
202+
cbIDEComment.foreground = UIUtil.getErrorForeground()
203+
cbIDEComment.text =
204+
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out. Check the command log for more details."
205+
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
206+
},
207+
onCountdown = { remainingMs ->
208+
cbIDE.renderer = IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.retrieve-ides.failed.retry", humanizeDuration(remainingMs)))
209+
},
210+
)
213211
withContext(Dispatchers.Main) {
214212
ideComboBoxModel.addAll(ides)
215213
cbIDE.selectedIndex = 0
216214
}
215+
} catch (e: Exception) {
216+
if (isCancellation(e)) {
217+
logger.info("Connection canceled due to ${e.javaClass}")
218+
} else {
219+
logger.error("Failed to retrieve IDEs (will not retry)", e)
220+
cbIDEComment.foreground = UIUtil.getErrorForeground()
221+
cbIDEComment.text = e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
222+
cbIDE.renderer = IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.retrieve-ides.failed"), UIUtil.getBalloonErrorIcon())
223+
}
217224
}
218225
}
219226
}

‎src/main/resources/messages/CoderGatewayBundle.properties

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@ gateway.connector.view.workspaces.token.comment=The last used token is shown abo
2828
gateway.connector.view.workspaces.token.rejected=This token was rejected.
2929
gateway.connector.view.workspaces.token.injected=This token was pulled from your CLI config.
3030
gateway.connector.view.workspaces.token.none=No existing token found.
31-
gateway.connector.view.coder.remoteproject.loading.text=Retrieving products...
32-
gateway.connector.view.coder.remoteproject.retry.text=Retrieving products (attempt {0})...
33-
gateway.connector.view.coder.remoteproject.error.text=Failed to retrieve IDEs
34-
gateway.connector.view.coder.remoteproject.retry-error.text=Failed to retrieve IDEs...retrying {0}
31+
gateway.connector.view.coder.retrieve-ides=Retrieving IDEs...
32+
gateway.connector.view.coder.retrieve.ides.retry=Retrieving IDEs (attempt {0})...
33+
gateway.connector.view.coder.retrieve-ides.failed=Failed to retrieve IDEs
34+
gateway.connector.view.coder.retrieve-ides.failed.retry=Failed to retrieve IDEs...retrying {0}
3535
gateway.connector.view.coder.remoteproject.next.text=Start IDE and connect
3636
gateway.connector.view.coder.remoteproject.choose.text=Choose IDE and project for workspace {0}
3737
gateway.connector.view.coder.remoteproject.ide.download.comment=This IDE will be downloaded from jetbrains.com and installed to the default path on the remote host.
@@ -42,10 +42,10 @@ gateway.connector.recentconnections.new.wizard.button.tooltip=Open a new Coder W
4242
gateway.connector.recentconnections.remove.button.tooltip=Remove from Recent Connections
4343
gateway.connector.recentconnections.terminal.button.tooltip=Open SSH Web Terminal
4444
gateway.connector.coder.connection.provider.title=Connecting to Coder workspace...
45-
gateway.connector.coder.connection.loading.text=Connecting...
46-
gateway.connector.coder.connection.retry.text=Connecting (attempt {0})...
47-
gateway.connector.coder.connection.retry-error.text=Failed to connect...retrying {0}
48-
gateway.connector.coder.connection.error.text=Failed to connect
45+
gateway.connector.coder.connecting=Connecting...
46+
gateway.connector.coder.connecting.retry=Connecting (attempt {0})...
47+
gateway.connector.coder.connection.failed=Failed to connect
48+
gateway.connector.coder.connecting.failed.retry=Failed to connect...retrying {0}
4949
gateway.connector.settings.binary-source.title=CLI source:
5050
gateway.connector.settings.binary-source.comment=Used to download the Coder \
5151
CLI which is necessary to make SSH connections. The If-None-Matched header \

0 commit comments

Comments
 (0)