surefire-reports.com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation-output.txt Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cloudpool.commons Show documentation
Show all versions of cloudpool.commons Show documentation
Common utility classes for building cloud pools.
09:28:08.002 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.002 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.003 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.004 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.004 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.004 [pool-29-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.005 [pool-29-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.006 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.006 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.010 [main] DEBUG [BaseCloudPool:stop:344] - stopping BaseCloudPool ...
09:28:08.010 [main] DEBUG [CachingPoolFetcher:close:120] - stopping CachingPoolFetcher ...
09:28:08.011 [main] DEBUG [StandardPoolUpdater:close:106] - shutting down StandardPoolUpdater ...
09:28:08.011 [main] INFO [BaseCloudPool:stop:350] - BaseCloudPool stopped.
09:28:08.011 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "NEWEST_INSTANCE",
"instanceHourMargin": 1800
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.011 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.012 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.013 [main] INFO [CachingPoolFetcher::83] - recovered cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.014 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.014 [pool-29-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.014 [pool-29-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.015 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.015 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.020 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.021 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.021 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.022 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.022 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.022 [pool-30-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.025 [pool-30-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.026 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.026 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.026 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.030 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.030 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.032 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.035 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.035 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.035 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 4
09:28:08.035 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.038 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.038 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.040 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.041 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 4
09:28:08.041 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.043 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 4 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.043 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=1, toSpare=0, toTerminate=[]}
09:28:08.043 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 1 new request(s)
09:28:08.044 [main] INFO [StandardPoolUpdater:startAlert:395] - 1 machine(s) were requested from cloud pool
09:28:08.046 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.049 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.053 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 300
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.053 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.054 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.054 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.055 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.055 [pool-31-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.056 [pool-31-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.056 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.057 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.057 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-22T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.057 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.058 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-22T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.058 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-22T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.059 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.059 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.059 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.059 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.059 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.060 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.061 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 0
09:28:08.061 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.062 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 0 (allocated: 1, active: 1), net size (excluding termination-queued): 1, termination queue: [ ]
09:28:08.062 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.062 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 1 evictable candidate(s)
09:28:08.062 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 1 candidate(s)
09:28:08.062 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-22T12:50:00.000Z}]}
09:28:08.063 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.063 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-1 for termination at 2014-04-22T12:50:00.000Z
09:28:08.063 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.063 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.064 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.064 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.065 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.066 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 0
09:28:08.066 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.067 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 0 (allocated: 1, active: 1), net size (excluding termination-queued): 0, termination queue: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.067 [main] DEBUG [ResizePlanner:calculateResizePlan:178] - desired size 0 equals net pool size, nothing to do
09:28:08.067 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[]}
09:28:08.067 [main] INFO [StandardPoolUpdater:doPoolUpdate:327] - pool is already properly sized (0)
09:28:08.067 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.068 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.068 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.068 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.069 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 0
09:28:08.069 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.070 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 0 (allocated: 1, active: 1), net size (excluding termination-queued): 0, termination queue: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.070 [main] DEBUG [ResizePlanner:calculateResizePlan:178] - desired size 0 equals net pool size, nothing to do
09:28:08.070 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[]}
09:28:08.070 [main] INFO [StandardPoolUpdater:doPoolUpdate:327] - pool is already properly sized (0)
09:28:08.070 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-1:2014-04-22T12:50:00.000Z ]
09:28:08.071 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-22T12:50:00.000Z}]
09:28:08.071 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.072 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-22T12:50:50.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.073 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-22T12:50:50.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-22T11:55:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.077 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.077 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.078 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.079 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.079 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.079 [pool-32-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.080 [pool-32-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.080 [pool-32-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.080 [pool-32-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.080 [pool-32-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.081 [pool-32-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.081 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.081 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.081 [main] DEBUG [CachingPoolFetcher:get:134] - no machine pool in cache yet. failing ...
09:28:08.081 [main] WARN [TestBaseCloudPoolOperation:desiredSize:1484] - no desired size could be retrieved: Could not serve machine pool: no fetch attempt has been successful yet. Latest error: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
09:28:08.081 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.081 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.082 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.083 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.083 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.083 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 0 (allocated: 0, effective: 0)
09:28:08.083 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 0
09:28:08.083 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: []
09:28:08.083 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 0 (allocated: 0, active: 0), net size (excluding termination-queued): 0, termination queue: [ ]
09:28:08.083 [main] DEBUG [ResizePlanner:calculateResizePlan:178] - desired size 0 equals net pool size, nothing to do
09:28:08.084 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[]}
09:28:08.084 [main] INFO [StandardPoolUpdater:doPoolUpdate:327] - pool is already properly sized (0)
09:28:08.084 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.084 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.087 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.087 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.091 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.092 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.093 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.093 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.093 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.094 [pool-34-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.095 [pool-34-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.096 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.096 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.097 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.098 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.098 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.099 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.101 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.101 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 2 (allocated: 3, effective: 2)
09:28:08.101 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.101 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.101 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.102 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.103 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 1
09:28:08.104 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.105 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 1 (allocated: 3, active: 2), net size (excluding termination-queued): 2, termination queue: [ ]
09:28:08.105 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.105 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 2 evictable candidate(s)
09:28:08.105 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 2 candidate(s)
09:28:08.106 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.106 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.106 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-2 for termination at 2014-04-17T12:00:00.000Z
09:28:08.106 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-2:2014-04-17T12:00:00.000Z ]
09:28:08.106 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-2:2014-04-17T12:00:00.000Z ]
09:28:08.107 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.107 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.108 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.110 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.114 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.114 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.115 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.115 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.116 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.116 [pool-35-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.117 [pool-35-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'RuntimeException: api outage'
09:28:08.117 [pool-35-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'RuntimeException: api outage'
09:28:08.117 [pool-35-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'RuntimeException: api outage'
09:28:08.117 [pool-35-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'RuntimeException: api outage'
09:28:08.118 [pool-35-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: java.lang.RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.118 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.118 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.118 [main] DEBUG [CachingPoolFetcher:get:134] - no machine pool in cache yet. failing ...
09:28:08.121 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.121 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.122 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.123 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.123 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.123 [pool-36-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.125 [pool-36-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.126 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.126 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.126 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.128 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.128 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.129 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.131 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.131 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.131 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 4
09:28:08.131 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.133 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.133 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.135 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.135 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 4
09:28:08.136 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.137 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 4 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.137 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=1, toSpare=0, toTerminate=[]}
09:28:08.137 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 1 new request(s)
09:28:08.138 [main] INFO [StandardPoolUpdater:startAlert:395] - 1 machine(s) were requested from cloud pool
09:28:08.138 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.140 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.143 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.144 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.144 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.145 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.145 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.145 [pool-37-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.147 [pool-37-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.148 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.148 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.148 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.149 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.150 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.151 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.152 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.153 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.153 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 5
09:28:08.153 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.154 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.154 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.155 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.156 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 5
09:28:08.156 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.158 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 5 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.158 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=2, toSpare=0, toTerminate=[]}
09:28:08.158 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 2 new request(s)
09:28:08.158 [main] WARN [StandardPoolUpdater:resize:138] - failed to resize machine pool webserver-pool: failure to complete request to start 2 machine(s) (0 machine(s) were launched): failed to add machines
com.elastisys.scale.cloudpool.commons.basepool.driver.StartMachinesException: failure to complete request to start 2 machine(s) (0 machine(s) were launched): failed to add machines
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.scaleOut(StandardPoolUpdater.java:339)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:314)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.completelyFailedScaleUpOfMachinePool(TestBaseCloudPoolOperation.java:406)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
Caused by: java.lang.Exception: failed to add machines
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.completelyFailedScaleUpOfMachinePool(TestBaseCloudPoolOperation.java:392)
... 25 common frames omitted
09:28:08.161 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.161 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.162 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.162 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.162 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.162 [pool-38-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.163 [pool-38-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.164 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.164 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.164 [main] INFO [StandardPoolUpdater:setServiceState:169] - service state OUT_OF_SERVICE assigned to i-1
09:28:08.168 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.168 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.169 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.169 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.170 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.170 [pool-40-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.171 [pool-40-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvid ... (truncated)'
09:28:08.172 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.172 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.172 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"REJECTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.174 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.174 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"REJECTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.176 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"REJECTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.177 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.177 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.180 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.180 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.181 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.181 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.181 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.181 [pool-41-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.182 [pool-41-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.182 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.182 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.183 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.183 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.183 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.184 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.184 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.184 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.184 [main] INFO [StandardPoolUpdater:terminateMachine:151] - terminating i-1
09:28:08.184 [main] DEBUG [StandardPoolUpdater:terminateMachine:158] - decrementing desiredSize to 0
09:28:08.185 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.185 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.189 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.189 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.190 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.190 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.190 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.190 [pool-42-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.191 [pool-42-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.192 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.192 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.192 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.192 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.192 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.193 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.193 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.193 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.193 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.194 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.194 [main] INFO [StandardPoolUpdater:detachMachine:207] - detaching i-1 from pool
09:28:08.195 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.199 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.199 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.200 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.200 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.200 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.200 [pool-43-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.202 [pool-43-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.203 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.203 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.203 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.205 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.206 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.207 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.209 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 4
09:28:08.209 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 4 (allocated: 6, effective: 4)
09:28:08.210 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.214 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.215 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.215 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.216 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.216 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.216 [pool-44-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.217 [pool-44-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.218 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.218 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.218 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.219 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.219 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.220 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.221 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.221 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.221 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.222 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.222 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.223 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.224 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 1
09:28:08.224 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.225 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 1 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.225 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 2 victim(s) to reach desired size
09:28:08.225 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.225 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 2 victim(s) from 3 candidate(s)
09:28:08.226 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.226 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 2 machine(s) for termination
09:28:08.226 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.227 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-2 for termination at 2014-04-17T12:00:00.000Z
09:28:08.227 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.227 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.227 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 2 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.228 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 2 machine(s) were terminated in cloud pool
09:28:08.228 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.230 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.233 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.233 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.234 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.234 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.234 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.234 [pool-45-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.235 [pool-45-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.235 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.236 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.236 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.236 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.237 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.237 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.238 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.238 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.238 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 5
09:28:08.238 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.239 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.239 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.240 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.240 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 5
09:28:08.240 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.241 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 5 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.241 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=2, toSpare=0, toTerminate=[]}
09:28:08.241 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 2 new request(s)
09:28:08.241 [main] INFO [StandardPoolUpdater:startAlert:395] - 2 machine(s) were requested from cloud pool
09:28:08.242 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.243 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.244 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.245 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.245 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.245 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.245 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.246 [pool-46-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.246 [pool-46-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'RuntimeException: api outage'
09:28:08.246 [pool-46-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'RuntimeException: api outage'
09:28:08.246 [pool-46-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'RuntimeException: api outage'
09:28:08.246 [pool-46-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'RuntimeException: api outage'
09:28:08.247 [pool-46-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: java.lang.RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.247 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.247 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.247 [main] DEBUG [CachingPoolFetcher:get:134] - no machine pool in cache yet. failing ...
09:28:08.249 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.250 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.250 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.250 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.250 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.250 [pool-47-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.251 [pool-47-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.252 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.252 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.252 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.253 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.253 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.254 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.254 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.255 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.255 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.255 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.255 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.256 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 3
09:28:08.256 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.257 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 3 (allocated: 2, active: 2), net size (excluding termination-queued): 2, termination queue: [ ]
09:28:08.257 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=1, toSpare=0, toTerminate=[]}
09:28:08.257 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 1 new request(s)
09:28:08.257 [main] INFO [StandardPoolUpdater:startAlert:395] - 1 machine(s) were requested from cloud pool
09:28:08.257 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.259 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.288 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.289 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.289 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.290 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.290 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.290 [pool-50-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.291 [pool-50-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.291 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.292 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.292 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"REQUESTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"REJECTED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-5","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.295 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.295 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.296 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.296 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.296 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.296 [pool-51-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.297 [pool-51-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.297 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.298 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.298 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.298 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.298 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.298 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.298 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.299 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.299 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.299 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.300 [main] INFO [StandardPoolUpdater:terminateMachine:151] - terminating i-1
09:28:08.300 [main] DEBUG [StandardPoolUpdater:terminateMachine:158] - decrementing desiredSize to 0
09:28:08.300 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.300 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.301 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.304 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.304 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.306 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.306 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.308 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.308 [pool-52-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.309 [pool-52-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.310 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.310 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.314 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.314 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.315 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.315 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.315 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.315 [pool-53-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.317 [pool-53-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.318 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.318 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.318 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.320 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.320 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.321 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.322 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 4
09:28:08.322 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 4 (allocated: 4, effective: 4)
09:28:08.322 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-8","machineState":"TERMINATING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-9","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.326 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.326 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.336 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.336 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.336 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.336 [pool-56-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.337 [pool-56-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.338 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.338 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.338 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.339 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.339 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.339 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.340 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.340 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.340 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 5
09:28:08.340 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.340 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.341 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.341 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 5
09:28:08.341 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.342 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 5 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.342 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=2, toSpare=0, toTerminate=[]}
09:28:08.342 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 2 new request(s)
09:28:08.342 [main] INFO [StandardPoolUpdater:startAlert:395] - 1 machine(s) were requested from cloud pool
09:28:08.343 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.344 [main] WARN [StandardPoolUpdater:resize:138] - failed to resize machine pool webserver-pool: failure to complete request to start 2 machine(s) (1 machine(s) were launched): failed to start second machine
com.elastisys.scale.cloudpool.commons.basepool.driver.StartMachinesException: failure to complete request to start 2 machine(s) (1 machine(s) were launched): failed to start second machine
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.scaleOut(StandardPoolUpdater.java:339)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:314)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.partiallyFailedScaleUpOfMachinePool(TestBaseCloudPoolOperation.java:451)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
Caused by: java.lang.Exception: failed to start second machine
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.partiallyFailedScaleUpOfMachinePool(TestBaseCloudPoolOperation.java:438)
... 25 common frames omitted
09:28:08.344 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.347 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.348 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.348 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.348 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.348 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.348 [pool-57-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.349 [pool-57-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.350 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.350 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.350 [main] INFO [StandardPoolUpdater:setMembershipStatus:177] - membership status {active=true, evictable=false} assigned to i-1
09:28:08.352 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.352 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.353 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.353 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.353 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.353 [pool-58-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.354 [pool-58-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvid ... (truncated)'
09:28:08.355 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.355 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.355 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.356 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.356 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.356 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.356 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.356 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 0 (allocated: 2, effective: 0)
09:28:08.357 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-6","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-7","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.358 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.359 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.359 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.359 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.359 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.360 [pool-59-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.360 [pool-59-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.361 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.361 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.361 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.361 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.361 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.362 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.362 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.362 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.362 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.362 [main] INFO [StandardPoolUpdater:attachMachine:189] - attaching instance i-1 to pool
09:28:08.362 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.363 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.365 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.365 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.366 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.366 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.366 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.366 [pool-60-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.367 [pool-60-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.368 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.368 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.368 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.368 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.368 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.369 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.369 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.370 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 2 (allocated: 2, effective: 2)
09:28:08.370 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.370 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.370 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.371 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.371 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.372 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.372 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.372 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.372 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 3 candidate(s)
09:28:08.372 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.372 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.372 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.372 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.372 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.372 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.373 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.373 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.374 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.376 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.377 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.377 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.377 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.377 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.378 [pool-61-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.378 [pool-61-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.379 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.379 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.379 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.379 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.379 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.380 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.380 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.380 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.380 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.381 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.381 [main] INFO [StandardPoolUpdater:terminateMachine:151] - terminating i-1
09:28:08.381 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.382 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.385 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.386 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.386 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.387 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.387 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.387 [pool-62-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.388 [pool-62-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.389 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.389 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.389 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.390 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.390 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.391 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.392 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.392 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.392 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.392 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.392 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.393 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.394 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.394 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.396 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.396 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.396 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.396 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 3 candidate(s)
09:28:08.397 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.397 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.397 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.397 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.397 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.397 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.398 [main] WARN [StandardPoolUpdater:terminateOverdueMachines:373] - failed to terminate instance 'i-3': terminate failed
com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: terminate failed
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.terminateOverdueMachines(StandardPoolUpdater.java:362)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:330)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.failedSingleMachineScaleDownOfMachinePool(TestBaseCloudPoolOperation.java:596)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
09:28:08.399 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.404 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.404 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.405 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.405 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.405 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.405 [pool-64-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.406 [pool-64-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.406 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.407 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.407 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.407 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.407 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.408 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.408 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.408 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.409 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.409 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.409 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.409 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.410 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 1
09:28:08.410 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.410 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 1 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.411 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 2 victim(s) to reach desired size
09:28:08.411 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.411 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 2 victim(s) from 3 candidate(s)
09:28:08.411 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.411 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 2 machine(s) for termination
09:28:08.411 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.411 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-2 for termination at 2014-04-17T12:00:00.000Z
09:28:08.411 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.411 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.411 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 2 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.412 [main] WARN [StandardPoolUpdater:terminateOverdueMachines:373] - failed to terminate instance 'i-3': terminate failed
com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: terminate failed
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.terminateOverdueMachines(StandardPoolUpdater.java:362)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:330)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.partiallyFailedScaleDownOfMachinePool(TestBaseCloudPoolOperation.java:671)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
09:28:08.412 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.412 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.415 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.418 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.418 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.419 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.419 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.419 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.419 [pool-67-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.420 [pool-67-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'RuntimeException: api outage'
09:28:08.420 [pool-67-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'RuntimeException: api outage'
09:28:08.420 [pool-67-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'RuntimeException: api outage'
09:28:08.420 [pool-67-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'RuntimeException: api outage'
09:28:08.420 [pool-67-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: java.lang.RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.420 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.421 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.421 [main] DEBUG [CachingPoolFetcher:get:134] - no machine pool in cache yet. failing ...
09:28:08.423 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.423 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.424 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.424 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.424 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.424 [pool-68-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.425 [pool-68-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.426 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.426 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.426 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.427 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.427 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.427 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.428 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 4
09:28:08.428 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 4 (allocated: 4, effective: 4)
09:28:08.428 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.430 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.430 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.431 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.431 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.431 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.431 [pool-69-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.432 [pool-69-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.432 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.432 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.432 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.433 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.433 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.433 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.433 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.433 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 0 (allocated: 0, effective: 0)
09:28:08.433 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.435 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.435 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.436 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.436 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.436 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.436 [pool-70-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.437 [pool-70-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.438 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.438 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.438 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.439 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.439 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.440 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.441 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.441 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.441 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.441 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.441 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.442 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.442 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 1
09:28:08.442 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.443 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 1 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.443 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 2 victim(s) to reach desired size
09:28:08.443 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.443 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 2 victim(s) from 3 candidate(s)
09:28:08.443 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.443 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 2 machine(s) for termination
09:28:08.443 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.444 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-2 for termination at 2014-04-17T12:00:00.000Z
09:28:08.444 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.444 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z i-2:2014-04-17T12:00:00.000Z ]
09:28:08.444 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 2 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}, ScheduledTermination{instance={"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.444 [main] WARN [StandardPoolUpdater:terminateOverdueMachines:373] - failed to terminate instance 'i-3': terminate failed
com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: terminate failed
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.terminateOverdueMachines(StandardPoolUpdater.java:362)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:330)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.failedMultiMachineScaleDownOfMachinePool(TestBaseCloudPoolOperation.java:633)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
09:28:08.445 [main] WARN [StandardPoolUpdater:terminateOverdueMachines:373] - failed to terminate instance 'i-2': terminate failed
com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: terminate failed
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.terminateOverdueMachines(StandardPoolUpdater.java:362)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.doPoolUpdate(StandardPoolUpdater.java:330)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:287)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.failedMultiMachineScaleDownOfMachinePool(TestBaseCloudPoolOperation.java:633)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
09:28:08.447 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.449 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.449 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.450 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.450 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.450 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.450 [pool-71-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.451 [pool-71-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.452 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.452 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.452 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.453 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.453 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.453 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.454 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.454 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.454 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.454 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.454 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.454 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.455 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.455 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.455 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.456 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.456 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.456 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 3 candidate(s)
09:28:08.456 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.456 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.456 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-3 for termination at 2014-04-17T12:00:00.000Z
09:28:08.456 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.456 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-3:2014-04-17T12:00:00.000Z ]
09:28:08.456 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.457 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.457 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.458 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.460 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.460 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.460 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.461 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.461 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.461 [pool-72-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.461 [pool-72-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.462 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.462 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.462 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.462 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.462 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.462 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.462 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.463 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.463 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.463 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.463 [main] INFO [StandardPoolUpdater:detachMachine:207] - detaching i-1 from pool
09:28:08.463 [main] DEBUG [StandardPoolUpdater:detachMachine:214] - decrementing desiredSize to 0
09:28:08.463 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.463 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.465 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.465 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.465 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.466 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.466 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.466 [pool-73-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.466 [pool-73-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.466 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.467 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.467 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.467 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.467 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.467 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.467 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.467 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.467 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.468 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.468 [main] INFO [StandardPoolUpdater:detachMachine:207] - detaching i-1 from pool
09:28:08.468 [main] DEBUG [StandardPoolUpdater:detachMachine:214] - decrementing desiredSize to 0
09:28:08.468 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.468 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.470 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.470 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.471 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.471 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.471 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.471 [pool-74-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.472 [pool-74-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.472 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.472 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.473 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.473 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.473 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.473 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.474 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.474 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 2 (allocated: 2, effective: 2)
09:28:08.474 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.474 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.475 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.475 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.475 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.476 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 2, active: 2), net size (excluding termination-queued): 2, termination queue: [ ]
09:28:08.476 [main] DEBUG [ResizePlanner:calculateResizePlan:178] - desired size 2 equals net pool size, nothing to do
09:28:08.476 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[]}
09:28:08.476 [main] INFO [StandardPoolUpdater:doPoolUpdate:327] - pool is already properly sized (2)
09:28:08.476 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.478 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.478 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.479 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.479 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.479 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.479 [pool-75-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.480 [pool-75-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.480 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.480 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.480 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.480 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.480 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.481 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.481 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.481 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 2 (allocated: 2, effective: 2)
09:28:08.482 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.482 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.482 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.482 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvid ... (truncated)'
09:28:08.483 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.483 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.483 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 2, active: 1), net size (excluding termination-queued): 1, termination queue: [ ]
09:28:08.483 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=1, toSpare=0, toTerminate=[]}
09:28:08.483 [main] INFO [StandardPoolUpdater:scaleOut:334] - sparing 0 machine(s) from termination, placing 1 new request(s)
09:28:08.483 [main] INFO [StandardPoolUpdater:startAlert:395] - 1 machine(s) were requested from cloud pool
09:28:08.483 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":false,"evictable":false},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.484 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ ]
09:28:08.486 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.487 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.487 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.487 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.487 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.487 [pool-77-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.488 [pool-77-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.488 [pool-77-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.488 [pool-77-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.488 [pool-77-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.488 [pool-77-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.488 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.488 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.489 [main] DEBUG [CachingPoolFetcher:get:134] - no machine pool in cache yet. failing ...
09:28:08.489 [main] WARN [TestBaseCloudPoolOperation:desiredSize:1484] - no desired size could be retrieved: Could not serve machine pool: no fetch attempt has been successful yet. Latest error: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
09:28:08.489 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.489 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.489 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.489 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.489 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.489 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'CloudPoolDriverException: cloud provider API outage'
09:28:08.489 [main] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.get(CachingPoolFetcher.java:129)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:279)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.doPoolUpdateWhenCloudPoolIsUnreachable(TestBaseCloudPoolOperation.java:895)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 32 common frames omitted
Caused by: com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 33 common frames omitted
09:28:08.490 [main] WARN [StandardPoolUpdater:resize:138] - failed to resize machine pool webserver-pool: machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
com.elastisys.scale.cloudpool.api.CloudPoolException: machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:217)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.get(CachingPoolFetcher.java:129)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.updateMachinePool(StandardPoolUpdater.java:279)
at com.elastisys.scale.cloudpool.commons.basepool.poolupdater.impl.StandardPoolUpdater.resize(StandardPoolUpdater.java:132)
at com.elastisys.scale.cloudpool.commons.basepool.BaseCloudPool.updateMachinePool(BaseCloudPool.java:464)
at com.elastisys.scale.cloudpool.commons.basepool.TestBaseCloudPoolOperation.doPoolUpdateWhenCloudPoolIsUnreachable(TestBaseCloudPoolOperation.java:895)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:367)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:274)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:161)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:290)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:242)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:121)
Caused by: com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
... 30 common frames omitted
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 32 common frames omitted
Caused by: com.elastisys.scale.cloudpool.commons.basepool.driver.CloudPoolDriverException: cloud provider API outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 33 common frames omitted
09:28:08.492 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.493 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.493 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.493 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.493 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.493 [pool-79-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.494 [pool-79-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.494 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.494 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.494 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.494 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.494 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.494 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[]}
09:28:08.494 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.494 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 0 (allocated: 0, effective: 0)
09:28:08.496 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.497 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.497 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.498 [main] INFO [CachingPoolFetcher::83] - recovered cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.498 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.498 [pool-81-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.498 [pool-81-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: 'RuntimeException: api outage'
09:28:08.498 [pool-81-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 2: 'RuntimeException: api outage'
09:28:08.499 [pool-81-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 3: 'RuntimeException: api outage'
09:28:08.499 [pool-81-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 4: 'RuntimeException: api outage'
09:28:08.499 [pool-81-thread-1] WARN [CachingPoolFetcher:refreshCache:216] - machine pool refresh failed: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
com.elastisys.scale.cloudpool.api.CloudPoolException: gave up trying to fetch pool members: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:95)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.get(RetryingPoolFetcher.java:76)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher.refreshCache(CachingPoolFetcher.java:209)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.CachingPoolFetcher$PoolRefreshTask.run(CachingPoolFetcher.java:234)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.elastisys.scale.commons.net.retryable.GaveUpException: gave up waiting for pool-fetch: result from last attempt: RuntimeException: api outage
at com.elastisys.scale.commons.net.retryable.Retryable.giveUp(Retryable.java:252)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:225)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher.listMachines(RetryingPoolFetcher.java:93)
... 10 common frames omitted
Caused by: java.lang.RuntimeException: api outage
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:108)
at com.elastisys.scale.cloudpool.commons.basepool.poolfetcher.impl.RetryingPoolFetcher$GetMachinePool.call(RetryingPoolFetcher.java:99)
at com.elastisys.scale.commons.net.retryable.Retryable.call(Retryable.java:212)
... 11 common frames omitted
09:28:08.499 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.499 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.499 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T12:00:00.000Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.501 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.501 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.501 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.502 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.502 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.502 [pool-82-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.502 [pool-82-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.502 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.503 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.503 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.503 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.503 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.503 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.503 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 1
09:28:08.503 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 1 (allocated: 1, effective: 1)
09:28:08.503 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.504 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.504 [main] INFO [StandardPoolUpdater:terminateMachine:151] - terminating i-1
09:28:08.504 [main] DEBUG [StandardPoolUpdater:terminateMachine:158] - decrementing desiredSize to 0
09:28:08.504 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 0
09:28:08.504 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.505 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.506 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "OLDEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.506 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.507 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.507 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.507 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.507 [pool-83-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.507 [pool-83-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[]'
09:28:08.508 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.508 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.508 [main] DEBUG [BaseCloudPool:stop:344] - stopping BaseCloudPool ...
09:28:08.508 [main] DEBUG [CachingPoolFetcher:close:120] - stopping CachingPoolFetcher ...
09:28:08.508 [main] DEBUG [StandardPoolUpdater:close:106] - shutting down StandardPoolUpdater ...
09:28:08.509 [main] INFO [BaseCloudPool:stop:350] - BaseCloudPool stopped.
09:28:08.509 [main] INFO [BaseCloudPool:stop:350] - BaseCloudPool stopped.
09:28:08.511 [main] DEBUG [BaseCloudPool:configure:282] - setting new configuration: {
"name": "webserver-pool",
"cloudApiSettings": {
"apiUser": "foo",
"apiPassword": "secret"
},
"provisioningTemplate": {
"size": "medium",
"image": "ubuntu-16.04"
},
"scaleInConfig": {
"victimSelectionPolicy": "NEWEST_INSTANCE",
"instanceHourMargin": 0
},
"poolFetch": {
"retries": {
"maxRetries": 3,
"initialBackoffDelay": {
"time": 0,
"unit": "seconds"
}
},
"refreshInterval": {
"time": 20,
"unit": "seconds"
},
"reachabilityTimeout": {
"time": 5,
"unit": "minutes"
}
}
}
09:28:08.512 [main] DEBUG [MultiplexingAlerter:registerAlerters:117] - no alert handlers registered.
09:28:08.512 [main] INFO [BaseCloudPool:start:326] - starting BaseCloudPool driving a CloudPoolDriver$$EnhancerByMockitoWithCGLIB$$5a37de39
09:28:08.512 [main] INFO [CachingPoolFetcher::85] - no previously stored machine pool found.
09:28:08.512 [main] DEBUG [CachingPoolFetcher::92] - started CachingPoolFetcher
09:28:08.512 [pool-85-thread-1] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.513 [pool-85-thread-1] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.513 [main] DEBUG [StandardPoolUpdater::100] - started StandardPoolUpdater
09:28:08.513 [main] INFO [BaseCloudPool:start:338] - BaseCloudPool started.
09:28:08.514 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.514 [main] DEBUG [StandardPoolUpdater:ensureDesiredSizeSet:253] - determining initial desired pool size ...
09:28:08.514 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.515 [main] DEBUG [StandardPoolUpdater:setDesiredSizeIfUnset:417] - determining initial desired size from pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.515 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 3
09:28:08.515 [main] INFO [StandardPoolUpdater:setDesiredSizeIfUnset:423] - initial desiredSize set to 3 (allocated: 3, effective: 3)
09:28:08.515 [main] INFO [StandardPoolUpdater:setDesiredSize:118] - set desiredSize to 2
09:28:08.515 [main] DEBUG [StandardPoolUpdater:updateMachinePool:275] - updating machine pool ...
09:28:08.515 [main] DEBUG [CachingPoolFetcher:refreshCache:207] - refreshing cached cloud pool ...
09:28:08.516 [main] DEBUG [Retryable:logResult:265] - pool-fetch: attempt 1: '[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider ... (truncated)'
09:28:08.516 [main] INFO [StandardPoolUpdater:doPoolUpdate:303] - updating pool size to desired size 2
09:28:08.516 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:305] - current pool members: [{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]
09:28:08.517 [main] DEBUG [ResizePlanner:calculateResizePlan:164] - desired pool size: 2 (allocated: 3, active: 3), net size (excluding termination-queued): 3, termination queue: [ ]
09:28:08.517 [main] DEBUG [ResizePlanner:scheduleForTermination:197] - need 1 victim(s) to reach desired size
09:28:08.517 [main] DEBUG [ResizePlanner:scheduleForTermination:200] - there are 3 evictable candidate(s)
09:28:08.517 [main] DEBUG [ResizePlanner:scheduleForTermination:203] - selecting 1 victim(s) from 3 candidate(s)
09:28:08.517 [main] DEBUG [ResizePlanner:calculateResizePlan:184] - suggested resize plan: ResizePlan{toRequest=0, toSpare=0, toTerminate=[ScheduledTermination{instance={"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]}
09:28:08.517 [main] INFO [StandardPoolUpdater:doPoolUpdate:318] - scheduling 1 machine(s) for termination
09:28:08.518 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:321] - scheduling machine i-1 for termination at 2014-04-17T12:00:00.000Z
09:28:08.518 [main] DEBUG [StandardPoolUpdater:doPoolUpdate:324] - termination queue: [ i-1:2014-04-17T12:00:00.000Z ]
09:28:08.518 [main] DEBUG [StandardPoolUpdater:terminateOverdueMachines:351] - checking termination queue for overdue machines: [ i-1:2014-04-17T12:00:00.000Z ]
09:28:08.518 [main] INFO [StandardPoolUpdater:terminateOverdueMachines:358] - terminating 1 overdue machine(s): [ScheduledTermination{instance={"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, terminationTime=2014-04-17T12:00:00.000Z}]
09:28:08.518 [main] INFO [StandardPoolUpdater:terminationAlert:451] - 1 machine(s) were terminated in cloud pool
09:28:08.518 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
09:28:08.519 [main] DEBUG [CachingPoolFetcher:get:145] - responding with cached machine pool: MachinePool{timestamp=2014-04-17T12:00:00.000Z, machines=[{"id":"i-1","machineState":"PENDING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.999Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-2","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.998Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-3","machineState":"RUNNING","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.997Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}, {"id":"i-4","machineState":"TERMINATED","serviceState":"UNKNOWN","membershipStatus":{"active":true,"evictable":true},"launchTime":"2014-04-17T11:59:59.996Z","publicIps":[],"privateIps":[],"cloudProvider":"AWS-EC2","region":"us-east-1","machineSize":"m1.small"}]}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy