src.com.android.server.PackageWatchdog Maven / Gradle / Ivy
Show all versions of android-all Show documentation
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server;
import static android.service.watchdog.ExplicitHealthCheckService.PackageConfig;
import static java.lang.annotation.RetentionPolicy.SOURCE;
import android.annotation.IntDef;
import android.annotation.Nullable;
import android.content.Context;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
import android.net.NetworkStackClient;
import android.os.Environment;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.provider.DeviceConfig;
import android.text.TextUtils;
import android.util.ArrayMap;
import android.util.ArraySet;
import android.util.AtomicFile;
import android.util.Slog;
import android.util.Xml;
import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.os.BackgroundThread;
import com.android.internal.util.FastXmlSerializer;
import com.android.internal.util.XmlUtils;
import libcore.io.IoUtils;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlSerializer;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.Retention;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* Monitors the health of packages on the system and notifies interested observers when packages
* fail. On failure, the registered observer with the least user impacting mitigation will
* be notified.
*/
public class PackageWatchdog {
private static final String TAG = "PackageWatchdog";
static final String PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS =
"watchdog_trigger_failure_duration_millis";
static final String PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT =
"watchdog_trigger_failure_count";
static final String PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED =
"watchdog_explicit_health_check_enabled";
// Duration to count package failures before it resets to 0
private static final int DEFAULT_TRIGGER_FAILURE_DURATION_MS =
(int) TimeUnit.MINUTES.toMillis(1);
// Number of package failures within the duration above before we notify observers
private static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5;
// Whether explicit health checks are enabled or not
private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true;
private static final int DB_VERSION = 1;
private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog";
private static final String TAG_PACKAGE = "package";
private static final String TAG_OBSERVER = "observer";
private static final String ATTR_VERSION = "version";
private static final String ATTR_NAME = "name";
private static final String ATTR_DURATION = "duration";
private static final String ATTR_EXPLICIT_HEALTH_CHECK_DURATION = "health-check-duration";
private static final String ATTR_PASSED_HEALTH_CHECK = "passed-health-check";
@GuardedBy("PackageWatchdog.class")
private static PackageWatchdog sPackageWatchdog;
private final Object mLock = new Object();
// System server context
private final Context mContext;
// Handler to run short running tasks
private final Handler mShortTaskHandler;
// Handler for processing IO and long running tasks
private final Handler mLongTaskHandler;
// Contains (observer-name -> observer-handle) that have ever been registered from
// previous boots. Observers with all packages expired are periodically pruned.
// It is saved to disk on system shutdown and repouplated on startup so it survives reboots.
@GuardedBy("mLock")
private final ArrayMap mAllObservers = new ArrayMap<>();
// File containing the XML data of monitored packages /data/system/package-watchdog.xml
private final AtomicFile mPolicyFile;
private final ExplicitHealthCheckController mHealthCheckController;
private final NetworkStackClient mNetworkStackClient;
@GuardedBy("mLock")
private boolean mIsPackagesReady;
// Flag to control whether explicit health checks are supported or not
@GuardedBy("mLock")
private boolean mIsHealthCheckEnabled = DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED;
@GuardedBy("mLock")
private int mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_DURATION_MS;
@GuardedBy("mLock")
private int mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
// SystemClock#uptimeMillis when we last executed #syncState
// 0 if no prune is scheduled.
@GuardedBy("mLock")
private long mUptimeAtLastStateSync;
private PackageWatchdog(Context context) {
// Needs to be constructed inline
this(context, new AtomicFile(
new File(new File(Environment.getDataDirectory(), "system"),
"package-watchdog.xml")),
new Handler(Looper.myLooper()), BackgroundThread.getHandler(),
new ExplicitHealthCheckController(context),
NetworkStackClient.getInstance());
}
/**
* Creates a PackageWatchdog that allows injecting dependencies.
*/
@VisibleForTesting
PackageWatchdog(Context context, AtomicFile policyFile, Handler shortTaskHandler,
Handler longTaskHandler, ExplicitHealthCheckController controller,
NetworkStackClient networkStackClient) {
mContext = context;
mPolicyFile = policyFile;
mShortTaskHandler = shortTaskHandler;
mLongTaskHandler = longTaskHandler;
mHealthCheckController = controller;
mNetworkStackClient = networkStackClient;
loadFromFile();
}
/** Creates or gets singleton instance of PackageWatchdog. */
public static PackageWatchdog getInstance(Context context) {
synchronized (PackageWatchdog.class) {
if (sPackageWatchdog == null) {
sPackageWatchdog = new PackageWatchdog(context);
}
return sPackageWatchdog;
}
}
/**
* Called during boot to notify when packages are ready on the device so we can start
* binding.
*/
public void onPackagesReady() {
synchronized (mLock) {
mIsPackagesReady = true;
mHealthCheckController.setCallbacks(packageName -> onHealthCheckPassed(packageName),
packages -> onSupportedPackages(packages),
() -> syncRequestsAsync());
setPropertyChangedListenerLocked();
updateConfigs();
registerNetworkStackHealthListener();
}
}
/**
* Registers {@code observer} to listen for package failures
*
* Observers are expected to call this on boot. It does not specify any packages but
* it will resume observing any packages requested from a previous boot.
*/
public void registerHealthObserver(PackageHealthObserver observer) {
synchronized (mLock) {
ObserverInternal internalObserver = mAllObservers.get(observer.getName());
if (internalObserver != null) {
internalObserver.mRegisteredObserver = observer;
}
}
}
/**
* Starts observing the health of the {@code packages} for {@code observer} and notifies
* {@code observer} of any package failures within the monitoring duration.
*
*
If monitoring a package supporting explicit health check, at the end of the monitoring
* duration if {@link #onHealthCheckPassed} was never called,
* {@link PackageHealthObserver#execute} will be called as if the package failed.
*
*
If {@code observer} is already monitoring a package in {@code packageNames},
* the monitoring window of that package will be reset to {@code durationMs} and the health
* check state will be reset to a default depending on if the package is contained in
* {@link mPackagesWithExplicitHealthCheckEnabled}.
*
* @throws IllegalArgumentException if {@code packageNames} is empty
* or {@code durationMs} is less than 1
*/
public void startObservingHealth(PackageHealthObserver observer, List packageNames,
long durationMs) {
if (packageNames.isEmpty()) {
Slog.wtf(TAG, "No packages to observe, " + observer.getName());
return;
}
if (durationMs < 1) {
// TODO: Instead of failing, monitor for default? 48hrs?
throw new IllegalArgumentException("Invalid duration " + durationMs + "ms for observer "
+ observer.getName() + ". Not observing packages " + packageNames);
}
List packages = new ArrayList<>();
for (int i = 0; i < packageNames.size(); i++) {
// Health checks not available yet so health check state will start INACTIVE
packages.add(new MonitoredPackage(packageNames.get(i), durationMs, false));
}
// Sync before we add the new packages to the observers. This will #pruneObservers,
// causing any elapsed time to be deducted from all existing packages before we add new
// packages. This maintains the invariant that the elapsed time for ALL (new and existing)
// packages is the same.
syncState("observing new packages");
synchronized (mLock) {
ObserverInternal oldObserver = mAllObservers.get(observer.getName());
if (oldObserver == null) {
Slog.d(TAG, observer.getName() + " started monitoring health "
+ "of packages " + packageNames);
mAllObservers.put(observer.getName(),
new ObserverInternal(observer.getName(), packages));
} else {
Slog.d(TAG, observer.getName() + " added the following "
+ "packages to monitor " + packageNames);
oldObserver.updatePackagesLocked(packages);
}
}
// Register observer in case not already registered
registerHealthObserver(observer);
// Sync after we add the new packages to the observers. We may have received packges
// requiring an earlier schedule than we are currently scheduled for.
syncState("updated observers");
}
/**
* Unregisters {@code observer} from listening to package failure.
* Additionally, this stops observing any packages that may have previously been observed
* even from a previous boot.
*/
public void unregisterHealthObserver(PackageHealthObserver observer) {
synchronized (mLock) {
mAllObservers.remove(observer.getName());
}
syncState("unregistering observer: " + observer.getName());
}
/**
* Returns packages observed by {@code observer}
*
* @return an empty set if {@code observer} has some packages observerd from a previous boot
* but has not registered itself in the current boot to receive notifications. Returns null
* if there are no active packages monitored from any boot.
*/
@Nullable
public Set getPackages(PackageHealthObserver observer) {
synchronized (mLock) {
for (int i = 0; i < mAllObservers.size(); i++) {
if (observer.getName().equals(mAllObservers.keyAt(i))) {
if (observer.equals(mAllObservers.valueAt(i).mRegisteredObserver)) {
return mAllObservers.valueAt(i).mPackages.keySet();
}
return Collections.emptySet();
}
}
}
return null;
}
/**
* Called when a process fails either due to a crash or ANR.
*
* For each package contained in the process, one registered observer with the least user
* impact will be notified for mitigation.
*
*
This method could be called frequently if there is a severe problem on the device.
*/
public void onPackageFailure(List packages) {
mLongTaskHandler.post(() -> {
synchronized (mLock) {
if (mAllObservers.isEmpty()) {
return;
}
for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
VersionedPackage versionedPackage = packages.get(pIndex);
// Observer that will receive failure for versionedPackage
PackageHealthObserver currentObserverToNotify = null;
int currentObserverImpact = Integer.MAX_VALUE;
// Find observer with least user impact
for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
ObserverInternal observer = mAllObservers.valueAt(oIndex);
PackageHealthObserver registeredObserver = observer.mRegisteredObserver;
if (registeredObserver != null
&& observer.onPackageFailureLocked(
versionedPackage.getPackageName())) {
int impact = registeredObserver.onHealthCheckFailed(versionedPackage);
if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE
&& impact < currentObserverImpact) {
currentObserverToNotify = registeredObserver;
currentObserverImpact = impact;
}
}
}
// Execute action with least user impact
if (currentObserverToNotify != null) {
currentObserverToNotify.execute(versionedPackage);
}
}
}
});
}
// TODO(b/120598832): Optimize write? Maybe only write a separate smaller file? Also
// avoid holding lock?
// This currently adds about 7ms extra to shutdown thread
/** Writes the package information to file during shutdown. */
public void writeNow() {
synchronized (mLock) {
// Must only run synchronous tasks as this runs on the ShutdownThread and no other
// thread is guaranteed to run during shutdown.
if (!mAllObservers.isEmpty()) {
mLongTaskHandler.removeCallbacks(this::saveToFileAsync);
pruneObserversLocked();
saveToFile();
Slog.i(TAG, "Last write to update package durations");
}
}
}
/**
* Enables or disables explicit health checks.
* If explicit health checks are enabled, the health check service is started.
*
If explicit health checks are disabled, pending explicit health check requests are
* passed and the health check service is stopped.
*/
private void setExplicitHealthCheckEnabled(boolean enabled) {
synchronized (mLock) {
mIsHealthCheckEnabled = enabled;
mHealthCheckController.setEnabled(enabled);
// Prune to update internal state whenever health check is enabled/disabled
syncState("health check state " + (enabled ? "enabled" : "disabled"));
}
}
/** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */
@Retention(SOURCE)
@IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_NONE,
PackageHealthObserverImpact.USER_IMPACT_LOW,
PackageHealthObserverImpact.USER_IMPACT_MEDIUM,
PackageHealthObserverImpact.USER_IMPACT_HIGH})
public @interface PackageHealthObserverImpact {
/** No action to take. */
int USER_IMPACT_NONE = 0;
/* Action has low user impact, user of a device will barely notice. */
int USER_IMPACT_LOW = 1;
/* Action has medium user impact, user of a device will likely notice. */
int USER_IMPACT_MEDIUM = 3;
/* Action has high user impact, a last resort, user of a device will be very frustrated. */
int USER_IMPACT_HIGH = 5;
}
/** Register instances of this interface to receive notifications on package failure. */
public interface PackageHealthObserver {
/**
* Called when health check fails for the {@code versionedPackage}.
*
* @return any one of {@link PackageHealthObserverImpact} to express the impact
* to the user on {@link #execute}
*/
@PackageHealthObserverImpact int onHealthCheckFailed(VersionedPackage versionedPackage);
/**
* Executes mitigation for {@link #onHealthCheckFailed}.
*
* @return {@code true} if action was executed successfully, {@code false} otherwise
*/
boolean execute(VersionedPackage versionedPackage);
// TODO(b/120598832): Ensure uniqueness?
/**
* Identifier for the observer, should not change across device updates otherwise the
* watchdog may drop observing packages with the old name.
*/
String getName();
}
long getTriggerFailureCount() {
synchronized (mLock) {
return mTriggerFailureCount;
}
}
/**
* Serializes and syncs health check requests with the {@link ExplicitHealthCheckController}.
*/
private void syncRequestsAsync() {
mShortTaskHandler.removeCallbacks(this::syncRequests);
mShortTaskHandler.post(this::syncRequests);
}
/**
* Syncs health check requests with the {@link ExplicitHealthCheckController}.
* Calls to this must be serialized.
*
* @see #syncRequestsAsync
*/
private void syncRequests() {
Set packages = null;
synchronized (mLock) {
if (mIsPackagesReady) {
packages = getPackagesPendingHealthChecksLocked();
} // else, we will sync requests when packages become ready
}
// Call outside lock to avoid holding lock when calling into the controller.
if (packages != null) {
Slog.i(TAG, "Syncing health check requests for packages: " + packages);
mHealthCheckController.syncRequests(packages);
}
}
/**
* Updates the observers monitoring {@code packageName} that explicit health check has passed.
*
* This update is strictly for registered observers at the time of the call
* Observers that register after this signal will have no knowledge of prior signals and will
* effectively behave as if the explicit health check hasn't passed for {@code packageName}.
*
*
{@code packageName} can still be considered failed if reported by
* {@link #onPackageFailureLocked} before the package expires.
*
*
Triggered by components outside the system server when they are fully functional after an
* update.
*/
private void onHealthCheckPassed(String packageName) {
Slog.i(TAG, "Health check passed for package: " + packageName);
boolean isStateChanged = false;
synchronized (mLock) {
for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) {
ObserverInternal observer = mAllObservers.valueAt(observerIdx);
MonitoredPackage monitoredPackage = observer.mPackages.get(packageName);
if (monitoredPackage != null) {
int oldState = monitoredPackage.getHealthCheckStateLocked();
int newState = monitoredPackage.tryPassHealthCheckLocked();
isStateChanged |= oldState != newState;
}
}
}
if (isStateChanged) {
syncState("health check passed for " + packageName);
}
}
private void onSupportedPackages(List supportedPackages) {
boolean isStateChanged = false;
Map supportedPackageTimeouts = new ArrayMap<>();
Iterator it = supportedPackages.iterator();
while (it.hasNext()) {
PackageConfig info = it.next();
supportedPackageTimeouts.put(info.getPackageName(), info.getHealthCheckTimeoutMillis());
}
synchronized (mLock) {
Slog.d(TAG, "Received supported packages " + supportedPackages);
Iterator oit = mAllObservers.values().iterator();
while (oit.hasNext()) {
Iterator pit = oit.next().mPackages.values().iterator();
while (pit.hasNext()) {
MonitoredPackage monitoredPackage = pit.next();
String packageName = monitoredPackage.getName();
int oldState = monitoredPackage.getHealthCheckStateLocked();
int newState;
if (supportedPackageTimeouts.containsKey(packageName)) {
// Supported packages become ACTIVE if currently INACTIVE
newState = monitoredPackage.setHealthCheckActiveLocked(
supportedPackageTimeouts.get(packageName));
} else {
// Unsupported packages are marked as PASSED unless already FAILED
newState = monitoredPackage.tryPassHealthCheckLocked();
}
isStateChanged |= oldState != newState;
}
}
}
if (isStateChanged) {
syncState("updated health check supported packages " + supportedPackages);
}
}
@GuardedBy("mLock")
private Set getPackagesPendingHealthChecksLocked() {
Slog.d(TAG, "Getting all observed packages pending health checks");
Set packages = new ArraySet<>();
Iterator oit = mAllObservers.values().iterator();
while (oit.hasNext()) {
ObserverInternal observer = oit.next();
Iterator pit =
observer.mPackages.values().iterator();
while (pit.hasNext()) {
MonitoredPackage monitoredPackage = pit.next();
String packageName = monitoredPackage.getName();
if (monitoredPackage.isPendingHealthChecksLocked()) {
packages.add(packageName);
}
}
}
return packages;
}
/**
* Syncs the state of the observers.
*
* Prunes all observers, saves new state to disk, syncs health check requests with the
* health check service and schedules the next state sync.
*/
private void syncState(String reason) {
synchronized (mLock) {
Slog.i(TAG, "Syncing state, reason: " + reason);
pruneObserversLocked();
saveToFileAsync();
syncRequestsAsync();
// Done syncing state, schedule the next state sync
scheduleNextSyncStateLocked();
}
}
private void syncStateWithScheduledReason() {
syncState("scheduled");
}
@GuardedBy("mLock")
private void scheduleNextSyncStateLocked() {
long durationMs = getNextStateSyncMillisLocked();
mShortTaskHandler.removeCallbacks(this::syncStateWithScheduledReason);
if (durationMs == Long.MAX_VALUE) {
Slog.i(TAG, "Cancelling state sync, nothing to sync");
mUptimeAtLastStateSync = 0;
} else {
Slog.i(TAG, "Scheduling next state sync in " + durationMs + "ms");
mUptimeAtLastStateSync = SystemClock.uptimeMillis();
mShortTaskHandler.postDelayed(this::syncStateWithScheduledReason, durationMs);
}
}
/**
* Returns the next duration in millis to sync the watchdog state.
*
* @returns Long#MAX_VALUE if there are no observed packages.
*/
@GuardedBy("mLock")
private long getNextStateSyncMillisLocked() {
long shortestDurationMs = Long.MAX_VALUE;
for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
ArrayMap packages = mAllObservers.valueAt(oIndex).mPackages;
for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
MonitoredPackage mp = packages.valueAt(pIndex);
long duration = mp.getShortestScheduleDurationMsLocked();
if (duration < shortestDurationMs) {
shortestDurationMs = duration;
}
}
}
return shortestDurationMs;
}
/**
* Removes {@code elapsedMs} milliseconds from all durations on monitored packages
* and updates other internal state.
*/
@GuardedBy("mLock")
private void pruneObserversLocked() {
long elapsedMs = mUptimeAtLastStateSync == 0
? 0 : SystemClock.uptimeMillis() - mUptimeAtLastStateSync;
if (elapsedMs <= 0) {
Slog.i(TAG, "Not pruning observers, elapsed time: " + elapsedMs + "ms");
return;
}
Slog.i(TAG, "Removing " + elapsedMs + "ms from all packages on all observers");
Iterator it = mAllObservers.values().iterator();
while (it.hasNext()) {
ObserverInternal observer = it.next();
Set failedPackages =
observer.prunePackagesLocked(elapsedMs);
if (!failedPackages.isEmpty()) {
onHealthCheckFailed(observer, failedPackages);
}
if (observer.mPackages.isEmpty()) {
Slog.i(TAG, "Discarding observer " + observer.mName + ". All packages expired");
it.remove();
}
}
}
private void onHealthCheckFailed(ObserverInternal observer,
Set failedPackages) {
mLongTaskHandler.post(() -> {
synchronized (mLock) {
PackageHealthObserver registeredObserver = observer.mRegisteredObserver;
if (registeredObserver != null) {
Iterator it = failedPackages.iterator();
while (it.hasNext()) {
String failedPackage = it.next().getName();
Slog.i(TAG, "Explicit health check failed for package " + failedPackage);
VersionedPackage versionedPkg = getVersionedPackage(failedPackage);
if (versionedPkg == null) {
Slog.w(TAG, "Explicit health check failed but could not find package "
+ failedPackage);
// TODO(b/120598832): Skip. We only continue to pass tests for now since
// the tests don't install any packages
versionedPkg = new VersionedPackage(failedPackage, 0L);
}
registeredObserver.execute(versionedPkg);
}
}
}
});
}
@Nullable
private VersionedPackage getVersionedPackage(String packageName) {
final PackageManager pm = mContext.getPackageManager();
if (pm == null) {
return null;
}
try {
final long versionCode = pm.getPackageInfo(
packageName, 0 /* flags */).getLongVersionCode();
return new VersionedPackage(packageName, versionCode);
} catch (PackageManager.NameNotFoundException e) {
return null;
}
}
/**
* Loads mAllObservers from file.
*
* Note that this is not thread safe and should only called be called
* from the constructor.
*/
private void loadFromFile() {
InputStream infile = null;
mAllObservers.clear();
try {
infile = mPolicyFile.openRead();
final XmlPullParser parser = Xml.newPullParser();
parser.setInput(infile, StandardCharsets.UTF_8.name());
XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG);
int outerDepth = parser.getDepth();
while (XmlUtils.nextElementWithin(parser, outerDepth)) {
ObserverInternal observer = ObserverInternal.read(parser, this);
if (observer != null) {
mAllObservers.put(observer.mName, observer);
}
}
} catch (FileNotFoundException e) {
// Nothing to monitor
} catch (IOException | NumberFormatException | XmlPullParserException e) {
Slog.wtf(TAG, "Unable to read monitored packages, deleting file", e);
mPolicyFile.delete();
} finally {
IoUtils.closeQuietly(infile);
}
}
/** Adds a {@link DeviceConfig#OnPropertiesChangedListener}. */
private void setPropertyChangedListenerLocked() {
DeviceConfig.addOnPropertiesChangedListener(
DeviceConfig.NAMESPACE_ROLLBACK,
mContext.getMainExecutor(),
(properties) -> {
if (!DeviceConfig.NAMESPACE_ROLLBACK.equals(properties.getNamespace())) {
return;
}
updateConfigs();
});
}
/**
* Health check is enabled or disabled after reading the flags
* from DeviceConfig.
*/
private void updateConfigs() {
synchronized (mLock) {
mTriggerFailureCount = DeviceConfig.getInt(
DeviceConfig.NAMESPACE_ROLLBACK,
PROPERTY_WATCHDOG_TRIGGER_FAILURE_COUNT,
DEFAULT_TRIGGER_FAILURE_COUNT);
if (mTriggerFailureCount <= 0) {
mTriggerFailureCount = DEFAULT_TRIGGER_FAILURE_COUNT;
}
mTriggerFailureDurationMs = DeviceConfig.getInt(
DeviceConfig.NAMESPACE_ROLLBACK,
PROPERTY_WATCHDOG_TRIGGER_DURATION_MILLIS,
DEFAULT_TRIGGER_FAILURE_DURATION_MS);
if (mTriggerFailureDurationMs <= 0) {
mTriggerFailureDurationMs = DEFAULT_TRIGGER_FAILURE_COUNT;
}
setExplicitHealthCheckEnabled(DeviceConfig.getBoolean(
DeviceConfig.NAMESPACE_ROLLBACK,
PROPERTY_WATCHDOG_EXPLICIT_HEALTH_CHECK_ENABLED,
DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED));
}
}
private void registerNetworkStackHealthListener() {
// TODO: have an internal method to trigger a rollback by reporting high severity errors,
// and rely on ActivityManager to inform the watchdog of severe network stack crashes
// instead of having this listener in parallel.
mNetworkStackClient.registerHealthListener(
packageName -> {
final VersionedPackage pkg = getVersionedPackage(packageName);
if (pkg == null) {
Slog.wtf(TAG, "NetworkStack failed but could not find its package");
return;
}
// This is a severe failure and recovery should be attempted immediately.
// TODO: have a better way to handle such failures.
final List pkgList = Collections.singletonList(pkg);
final long failureCount = getTriggerFailureCount();
for (int i = 0; i < failureCount; i++) {
onPackageFailure(pkgList);
}
});
}
/**
* Persists mAllObservers to file. Threshold information is ignored.
*/
private boolean saveToFile() {
Slog.i(TAG, "Saving observer state to file");
synchronized (mLock) {
FileOutputStream stream;
try {
stream = mPolicyFile.startWrite();
} catch (IOException e) {
Slog.w(TAG, "Cannot update monitored packages", e);
return false;
}
try {
XmlSerializer out = new FastXmlSerializer();
out.setOutput(stream, StandardCharsets.UTF_8.name());
out.startDocument(null, true);
out.startTag(null, TAG_PACKAGE_WATCHDOG);
out.attribute(null, ATTR_VERSION, Integer.toString(DB_VERSION));
for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) {
mAllObservers.valueAt(oIndex).writeLocked(out);
}
out.endTag(null, TAG_PACKAGE_WATCHDOG);
out.endDocument();
mPolicyFile.finishWrite(stream);
return true;
} catch (IOException e) {
Slog.w(TAG, "Failed to save monitored packages, restoring backup", e);
mPolicyFile.failWrite(stream);
return false;
} finally {
IoUtils.closeQuietly(stream);
}
}
}
private void saveToFileAsync() {
if (!mLongTaskHandler.hasCallbacks(this::saveToFile)) {
mLongTaskHandler.post(this::saveToFile);
}
}
/**
* Represents an observer monitoring a set of packages along with the failure thresholds for
* each package.
*
* Note, the PackageWatchdog#mLock must always be held when reading or writing
* instances of this class.
*/
//TODO(b/120598832): Remove 'm' from non-private fields
private static class ObserverInternal {
public final String mName;
//TODO(b/120598832): Add getter for mPackages
@GuardedBy("mLock")
public final ArrayMap mPackages = new ArrayMap<>();
@Nullable
@GuardedBy("mLock")
public PackageHealthObserver mRegisteredObserver;
ObserverInternal(String name, List packages) {
mName = name;
updatePackagesLocked(packages);
}
/**
* Writes important {@link MonitoredPackage} details for this observer to file.
* Does not persist any package failure thresholds.
*/
@GuardedBy("mLock")
public boolean writeLocked(XmlSerializer out) {
try {
out.startTag(null, TAG_OBSERVER);
out.attribute(null, ATTR_NAME, mName);
for (int i = 0; i < mPackages.size(); i++) {
MonitoredPackage p = mPackages.valueAt(i);
p.writeLocked(out);
}
out.endTag(null, TAG_OBSERVER);
return true;
} catch (IOException e) {
Slog.w(TAG, "Cannot save observer", e);
return false;
}
}
@GuardedBy("mLock")
public void updatePackagesLocked(List packages) {
for (int pIndex = 0; pIndex < packages.size(); pIndex++) {
MonitoredPackage p = packages.get(pIndex);
mPackages.put(p.mName, p);
}
}
/**
* Reduces the monitoring durations of all packages observed by this observer by
* {@code elapsedMs}. If any duration is less than 0, the package is removed from
* observation. If any health check duration is less than 0, the health check result
* is evaluated.
*
* @return a {@link Set} of packages that were removed from the observer without explicit
* health check passing, or an empty list if no package expired for which an explicit health
* check was still pending
*/
@GuardedBy("mLock")
private Set prunePackagesLocked(long elapsedMs) {
Set failedPackages = new ArraySet<>();
Iterator it = mPackages.values().iterator();
while (it.hasNext()) {
MonitoredPackage p = it.next();
int oldState = p.getHealthCheckStateLocked();
int newState = p.handleElapsedTimeLocked(elapsedMs);
if (oldState != MonitoredPackage.STATE_FAILED
&& newState == MonitoredPackage.STATE_FAILED) {
Slog.i(TAG, "Package " + p.mName + " failed health check");
failedPackages.add(p);
}
if (p.isExpiredLocked()) {
it.remove();
}
}
return failedPackages;
}
/**
* Increments failure counts of {@code packageName}.
* @returns {@code true} if failure threshold is exceeded, {@code false} otherwise
*/
@GuardedBy("mLock")
public boolean onPackageFailureLocked(String packageName) {
MonitoredPackage p = mPackages.get(packageName);
if (p != null) {
return p.onFailureLocked();
}
return false;
}
/**
* Returns one ObserverInternal from the {@code parser} and advances its state.
*
* Note that this method is not thread safe. It should only be called from
* #loadFromFile which in turn is only called on construction of the
* singleton PackageWatchdog.
**/
public static ObserverInternal read(XmlPullParser parser, PackageWatchdog watchdog) {
String observerName = null;
if (TAG_OBSERVER.equals(parser.getName())) {
observerName = parser.getAttributeValue(null, ATTR_NAME);
if (TextUtils.isEmpty(observerName)) {
Slog.wtf(TAG, "Unable to read observer name");
return null;
}
}
List packages = new ArrayList<>();
int innerDepth = parser.getDepth();
try {
while (XmlUtils.nextElementWithin(parser, innerDepth)) {
if (TAG_PACKAGE.equals(parser.getName())) {
try {
String packageName = parser.getAttributeValue(null, ATTR_NAME);
long duration = Long.parseLong(
parser.getAttributeValue(null, ATTR_DURATION));
long healthCheckDuration = Long.parseLong(
parser.getAttributeValue(null,
ATTR_EXPLICIT_HEALTH_CHECK_DURATION));
boolean hasPassedHealthCheck = Boolean.parseBoolean(
parser.getAttributeValue(null, ATTR_PASSED_HEALTH_CHECK));
if (!TextUtils.isEmpty(packageName)) {
packages.add(watchdog.new MonitoredPackage(packageName, duration,
healthCheckDuration, hasPassedHealthCheck));
}
} catch (NumberFormatException e) {
Slog.wtf(TAG, "Skipping package for observer " + observerName, e);
continue;
}
}
}
} catch (XmlPullParserException | IOException e) {
Slog.wtf(TAG, "Unable to read observer " + observerName, e);
return null;
}
if (packages.isEmpty()) {
return null;
}
return new ObserverInternal(observerName, packages);
}
}
/**
* Represents a package and its health check state along with the time
* it should be monitored for.
*
* Note, the PackageWatchdog#mLock must always be held when reading or writing
* instances of this class.
*/
class MonitoredPackage {
// Health check states
// TODO(b/120598832): Prefix with HEALTH_CHECK
// mName has not passed health check but has requested a health check
public static final int STATE_ACTIVE = 0;
// mName has not passed health check and has not requested a health check
public static final int STATE_INACTIVE = 1;
// mName has passed health check
public static final int STATE_PASSED = 2;
// mName has failed health check
public static final int STATE_FAILED = 3;
//TODO(b/120598832): VersionedPackage?
private final String mName;
// One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after
// methods that could change the health check state: handleElapsedTimeLocked and
// tryPassHealthCheckLocked
private int mHealthCheckState = STATE_INACTIVE;
// Whether an explicit health check has passed.
// This value in addition with mHealthCheckDurationMs determines the health check state
// of the package, see #getHealthCheckStateLocked
@GuardedBy("mLock")
private boolean mHasPassedHealthCheck;
// System uptime duration to monitor package.
@GuardedBy("mLock")
private long mDurationMs;
// System uptime duration to check the result of an explicit health check
// Initially, MAX_VALUE until we get a value from the health check service
// and request health checks.
// This value in addition with mHasPassedHealthCheck determines the health check state
// of the package, see #getHealthCheckStateLocked
@GuardedBy("mLock")
private long mHealthCheckDurationMs = Long.MAX_VALUE;
// System uptime of first package failure
@GuardedBy("mLock")
private long mUptimeStartMs;
// Number of failures since mUptimeStartMs
@GuardedBy("mLock")
private int mFailures;
MonitoredPackage(String name, long durationMs, boolean hasPassedHealthCheck) {
this(name, durationMs, Long.MAX_VALUE, hasPassedHealthCheck);
}
MonitoredPackage(String name, long durationMs, long healthCheckDurationMs,
boolean hasPassedHealthCheck) {
mName = name;
mDurationMs = durationMs;
mHealthCheckDurationMs = healthCheckDurationMs;
mHasPassedHealthCheck = hasPassedHealthCheck;
updateHealthCheckStateLocked();
}
/** Writes the salient fields to disk using {@code out}. */
@GuardedBy("mLock")
public void writeLocked(XmlSerializer out) throws IOException {
out.startTag(null, TAG_PACKAGE);
out.attribute(null, ATTR_NAME, mName);
out.attribute(null, ATTR_DURATION, String.valueOf(mDurationMs));
out.attribute(null, ATTR_EXPLICIT_HEALTH_CHECK_DURATION,
String.valueOf(mHealthCheckDurationMs));
out.attribute(null, ATTR_PASSED_HEALTH_CHECK,
String.valueOf(mHasPassedHealthCheck));
out.endTag(null, TAG_PACKAGE);
}
/**
* Increment package failures or resets failure count depending on the last package failure.
*
* @return {@code true} if failure count exceeds a threshold, {@code false} otherwise
*/
@GuardedBy("mLock")
public boolean onFailureLocked() {
final long now = SystemClock.uptimeMillis();
final long duration = now - mUptimeStartMs;
if (duration > mTriggerFailureDurationMs) {
// TODO(b/120598832): Reseting to 1 is not correct
// because there may be more than 1 failure in the last trigger window from now
// This is the RescueParty impl, will leave for now
mFailures = 1;
mUptimeStartMs = now;
} else {
mFailures++;
}
boolean failed = mFailures >= mTriggerFailureCount;
if (failed) {
mFailures = 0;
}
return failed;
}
/**
* Sets the initial health check duration.
*
* @return the new health check state
*/
@GuardedBy("mLock")
public int setHealthCheckActiveLocked(long initialHealthCheckDurationMs) {
if (initialHealthCheckDurationMs <= 0) {
Slog.wtf(TAG, "Cannot set non-positive health check duration "
+ initialHealthCheckDurationMs + "ms for package " + mName
+ ". Using total duration " + mDurationMs + "ms instead");
initialHealthCheckDurationMs = mDurationMs;
}
if (mHealthCheckState == STATE_INACTIVE) {
// Transitions to ACTIVE
mHealthCheckDurationMs = initialHealthCheckDurationMs;
}
return updateHealthCheckStateLocked();
}
/**
* Updates the monitoring durations of the package.
*
* @return the new health check state
*/
@GuardedBy("mLock")
public int handleElapsedTimeLocked(long elapsedMs) {
if (elapsedMs <= 0) {
Slog.w(TAG, "Cannot handle non-positive elapsed time for package " + mName);
return mHealthCheckState;
}
// Transitions to FAILED if now <= 0 and health check not passed
mDurationMs -= elapsedMs;
if (mHealthCheckState == STATE_ACTIVE) {
// We only update health check durations if we have #setHealthCheckActiveLocked
// This ensures we don't leave the INACTIVE state for an unexpected elapsed time
// Transitions to FAILED if now <= 0 and health check not passed
mHealthCheckDurationMs -= elapsedMs;
}
return updateHealthCheckStateLocked();
}
/**
* Marks the health check as passed and transitions to {@link #STATE_PASSED}
* if not yet {@link #STATE_FAILED}.
*
* @return the new health check state
*/
@GuardedBy("mLock")
public int tryPassHealthCheckLocked() {
if (mHealthCheckState != STATE_FAILED) {
// FAILED is a final state so only pass if we haven't failed
// Transition to PASSED
mHasPassedHealthCheck = true;
}
return updateHealthCheckStateLocked();
}
/** Returns the monitored package name. */
private String getName() {
return mName;
}
//TODO(b/120598832): IntDef
/**
* Returns the current health check state, any of {@link #STATE_ACTIVE},
* {@link #STATE_INACTIVE} or {@link #STATE_PASSED}
*/
@GuardedBy("mLock")
public int getHealthCheckStateLocked() {
return mHealthCheckState;
}
/**
* Returns the shortest duration before the package should be scheduled for a prune.
*
* @return the duration or {@link Long#MAX_VALUE} if the package should not be scheduled
*/
@GuardedBy("mLock")
public long getShortestScheduleDurationMsLocked() {
// Consider health check duration only if #isPendingHealthChecksLocked is true
return Math.min(toPositive(mDurationMs),
isPendingHealthChecksLocked()
? toPositive(mHealthCheckDurationMs) : Long.MAX_VALUE);
}
/**
* Returns {@code true} if the total duration left to monitor the package is less than or
* equal to 0 {@code false} otherwise.
*/
@GuardedBy("mLock")
public boolean isExpiredLocked() {
return mDurationMs <= 0;
}
/**
* Returns {@code true} if the package, {@link #getName} is expecting health check results
* {@code false} otherwise.
*/
@GuardedBy("mLock")
public boolean isPendingHealthChecksLocked() {
return mHealthCheckState == STATE_ACTIVE || mHealthCheckState == STATE_INACTIVE;
}
/**
* Updates the health check state based on {@link #mHasPassedHealthCheck}
* and {@link #mHealthCheckDurationMs}.
*
* @return the new health check state
*/
@GuardedBy("mLock")
private int updateHealthCheckStateLocked() {
int oldState = mHealthCheckState;
if (mHasPassedHealthCheck) {
// Set final state first to avoid ambiguity
mHealthCheckState = STATE_PASSED;
} else if (mHealthCheckDurationMs <= 0 || mDurationMs <= 0) {
// Set final state first to avoid ambiguity
mHealthCheckState = STATE_FAILED;
} else if (mHealthCheckDurationMs == Long.MAX_VALUE) {
mHealthCheckState = STATE_INACTIVE;
} else {
mHealthCheckState = STATE_ACTIVE;
}
Slog.i(TAG, "Updated health check state for package " + mName + ": "
+ toString(oldState) + " -> " + toString(mHealthCheckState));
return mHealthCheckState;
}
/** Returns a {@link String} representation of the current health check state. */
private String toString(int state) {
switch (state) {
case STATE_ACTIVE:
return "ACTIVE";
case STATE_INACTIVE:
return "INACTIVE";
case STATE_PASSED:
return "PASSED";
case STATE_FAILED:
return "FAILED";
default:
return "UNKNOWN";
}
}
/** Returns {@code value} if it is greater than 0 or {@link Long#MAX_VALUE} otherwise. */
private long toPositive(long value) {
return value > 0 ? value : Long.MAX_VALUE;
}
}
}