Viewing file:  numerics_alert.py (12.79 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Data structures and algorithms for numerics alert.
The alerts are generated when a Tensor's elements contain bad values,
including nan, -inf and +inf.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
import json
import re
import tensorflow as tf
from tensorflow.python import debug as tf_debug
from tensorboard.plugins.debugger import constants
# The following two namedtuples are the same except that
# 1) `timestamp` in `NumericsAlert` is the timestamp of a single alerting event,
# while `first_timestamp` in `NumericsAlertReportRow` is the first (earliest)
# timestamp of a se tof aggregated `NumericsAlert`s.
# 2) The counts in `NumericsAlert` are the counts of elements in the tensor,
# while the event counts in `NumericsAlertReportRow` are counts of previous
# `NumericsAlert` events of the corresponding categories.
NumericsAlert = collections.namedtuple(
"NumericsAlert",
["device_name", "tensor_name", "timestamp", "nan_count", "neg_inf_count",
"pos_inf_count"])
NumericsAlertReportRow = collections.namedtuple(
"NumericsAlertReportRow",
["device_name", "tensor_name", "first_timestamp", "nan_event_count",
"neg_inf_event_count", "pos_inf_event_count"])
# Used to reconstruct an _EventTracker from data read from disk. When updating
# this named tuple, make sure to keep the properties of _EventTracker in sync.
EventTrackerDescription = collections.namedtuple(
"EventTrackerDescription",
["event_count", "first_timestamp", "last_timestamp"])
# Used to reconstruct NumericsAlertHistory.
HistoryTriplet = collections.namedtuple(
"HistoryTriplet",
["device", "tensor", "jsonable_history"])
class _EventTracker(object):
"""Track events for a single category of values (NaN, -Inf, or +Inf)."""
def __init__(self, event_count=0, first_timestamp=-1, last_timestamp=-1):
"""Tracks events for a single category of values.
Args:
event_count: The initial event count to use.
first_timestamp: The timestamp of the first event with this value.
last_timestamp: The timestamp of the last event with this category of
values.
"""
# When updating the properties of this class, make sure to keep
# EventTrackerDescription in sync so that data can be written to and from
# disk correctly.
self.event_count = event_count
self.first_timestamp = first_timestamp
self.last_timestamp = last_timestamp
def add(self, timestamp):
if self.event_count == 0:
self.first_timestamp = timestamp
self.last_timestamp = timestamp
else:
if timestamp < self.first_timestamp:
self.first_timestamp = timestamp
if timestamp > self.last_timestamp:
self.last_timestamp = timestamp
self.event_count += 1
def get_description(self):
return EventTrackerDescription(
self.event_count, self.first_timestamp, self.last_timestamp)
class NumericsAlertHistory(object):
"""History of numerics alerts."""
def __init__(self, initialization_list=None):
"""Stores alert history for a single device, tensor pair.
Args:
initialization_list: (`list`) An optional list parsed from JSON read
from disk. That entity is used to initialize this NumericsAlertHistory.
Use the create_jsonable_object method of this class to create such an
object.
"""
if initialization_list:
# Use data to initialize this NumericsAlertHistory.
self._trackers = {}
for value_category_key, description_list in initialization_list.items():
description = EventTrackerDescription._make(description_list)
self._trackers[value_category_key] = _EventTracker(
event_count=description.event_count,
first_timestamp=description.first_timestamp,
last_timestamp=description.last_timestamp)
else:
# Start cleanly. With no prior data.
self._trackers = {
constants.NAN_KEY: _EventTracker(),
constants.NEG_INF_KEY: _EventTracker(),
constants.POS_INF_KEY: _EventTracker(),
}
def add(self, numerics_alert):
if numerics_alert.nan_count:
self._trackers[constants.NAN_KEY].add(numerics_alert.timestamp)
if numerics_alert.neg_inf_count:
self._trackers[constants.NEG_INF_KEY].add(numerics_alert.timestamp)
if numerics_alert.pos_inf_count:
self._trackers[constants.POS_INF_KEY].add(numerics_alert.timestamp)
def first_timestamp(self, event_key=None):
"""Obtain the first timestamp.
Args:
event_key: the type key of the sought events (e.g., constants.NAN_KEY).
If None, includes all event type keys.
Returns:
First (earliest) timestamp of all the events of the given type (or all
event types if event_key is None).
"""
if event_key is None:
timestamps = [self._trackers[key].first_timestamp
for key in self._trackers]
return min(timestamp for timestamp in timestamps if timestamp >= 0)
else:
return self._trackers[event_key].first_timestamp
def last_timestamp(self, event_key=None):
"""Obtain the last timestamp.
Args:
event_key: the type key of the sought events (e.g., constants.NAN_KEY). If
None, includes all event type keys.
Returns:
Last (latest) timestamp of all the events of the given type (or all
event types if event_key is None).
"""
if event_key is None:
timestamps = [self._trackers[key].first_timestamp
for key in self._trackers]
return max(timestamp for timestamp in timestamps if timestamp >= 0)
else:
return self._trackers[event_key].last_timestamp
def event_count(self, event_key):
"""Obtain event count.
Args:
event_key: the type key of the sought events (e.g., constants.NAN_KEY). If
None, includes all event type keys.
Returns:
If event_key is None, return the sum of the event_count of all event
types. Otherwise, return the event_count of the specified event type.
"""
return self._trackers[event_key].event_count
def create_jsonable_history(self):
"""Creates a JSON-able representation of this object.
Returns:
A dictionary mapping key to EventTrackerDescription (which can be used to
create event trackers).
"""
return {value_category_key: tracker.get_description()
for (value_category_key, tracker) in self._trackers.items()}
class NumericsAlertRegistry(object):
"""A registry for alerts on numerics (e.g., due to NaNs and infinities)."""
def __init__(self, capacity=100, initialization_list=None):
"""Constructor.
Args:
capacity: (`int`) maximum number of device-tensor keys to store.
initialization_list: (`list`) An optional list (parsed from JSON) that
is used to initialize the data within this registry. Use the
create_jsonable_registry method of NumericsAlertRegistry to create such
a list.
"""
self._capacity = capacity
# A map from device-tensor key to a the TensorAlertRecord namedtuple.
# The device-tensor key is a 2-tuple of the format (device_name, node_name).
# E.g., ("/job:worker/replica:0/task:1/gpu:0", "cross_entropy/Log:0").
self._data = dict()
if initialization_list:
# Initialize the alert registry using the data passed in. This might be
# backup data used to restore the registry after say a borg pre-emption.
for entry in initialization_list:
triplet = HistoryTriplet._make(entry)
self._data[(triplet.device, triplet.tensor)] = NumericsAlertHistory(
initialization_list=triplet.jsonable_history)
def register(self, numerics_alert):
"""Register an alerting numeric event.
Args:
numerics_alert: An instance of `NumericsAlert`.
"""
key = (numerics_alert.device_name, numerics_alert.tensor_name)
if key in self._data:
self._data[key].add(numerics_alert)
else:
if len(self._data) < self._capacity:
history = NumericsAlertHistory()
history.add(numerics_alert)
self._data[key] = history
def report(self, device_name_filter=None, tensor_name_filter=None):
"""Get a report of offending device/tensor names.
The report includes information about the device name, tensor name, first
(earliest) timestamp of the alerting events from the tensor, in addition to
counts of nan, positive inf and negative inf events.
Args:
device_name_filter: regex filter for device name, or None (not filtered).
tensor_name_filter: regex filter for tensor name, or None (not filtered).
Returns:
A list of NumericsAlertReportRow, sorted by the first_timestamp in
asecnding order.
"""
report = []
for key in self._data:
device_name, tensor_name = key
history = self._data[key]
report.append(
NumericsAlertReportRow(
device_name=device_name,
tensor_name=tensor_name,
first_timestamp=history.first_timestamp(),
nan_event_count=history.event_count(constants.NAN_KEY),
neg_inf_event_count=history.event_count(constants.NEG_INF_KEY),
pos_inf_event_count=history.event_count(constants.POS_INF_KEY)))
if device_name_filter:
device_name_pattern = re.compile(device_name_filter)
report = [item for item in report
if device_name_pattern.match(item.device_name)]
if tensor_name_filter:
tensor_name_pattern = re.compile(tensor_name_filter)
report = [item for item in report
if tensor_name_pattern.match(item.tensor_name)]
# Sort results chronologically.
return sorted(report, key=lambda x: x.first_timestamp)
def create_jsonable_registry(self):
"""Creates a JSON-able representation of this object.
Returns:
A dictionary mapping (device, tensor name) to JSON-able object
representations of NumericsAlertHistory.
"""
# JSON does not support tuples as keys. Only strings. Therefore, we store
# the device name, tensor name, and dictionary data within a 3-item list.
return [HistoryTriplet(pair[0], pair[1], history.create_jsonable_history())
for (pair, history) in self._data.items()]
def extract_numerics_alert(event):
"""Determines whether a health pill event contains bad values.
A bad value is one of NaN, -Inf, or +Inf.
Args:
event: (`Event`) A `tensorflow.Event` proto from `DebugNumericSummary`
ops.
Returns:
An instance of `NumericsAlert`, if bad values are found.
`None`, if no bad values are found.
Raises:
ValueError: if the event does not have the expected tag prefix or the
debug op name is not the expected debug op name suffix.
"""
value = event.summary.value[0]
debugger_plugin_metadata_content = None
if value.HasField("metadata"):
plugin_data = value.metadata.plugin_data
if plugin_data.plugin_name == constants.DEBUGGER_PLUGIN_NAME:
debugger_plugin_metadata_content = plugin_data.content
if not debugger_plugin_metadata_content:
raise ValueError("Event proto input lacks debugger plugin SummaryMetadata.")
debugger_plugin_metadata_content = tf.compat.as_text(
debugger_plugin_metadata_content)
try:
content_object = json.loads(debugger_plugin_metadata_content)
device_name = content_object["device"]
except (KeyError, ValueError) as e:
raise ValueError("Could not determine device from JSON string %r, %r" %
(debugger_plugin_metadata_content, e))
debug_op_suffix = ":DebugNumericSummary"
if not value.node_name.endswith(debug_op_suffix):
raise ValueError(
"Event proto input does not have the expected debug op suffix %s" %
debug_op_suffix)
tensor_name = value.node_name[:-len(debug_op_suffix)]
elements = tf_debug.load_tensor_from_event(event)
nan_count = elements[constants.NAN_NUMERIC_SUMMARY_OP_INDEX]
neg_inf_count = elements[constants.NEG_INF_NUMERIC_SUMMARY_OP_INDEX]
pos_inf_count = elements[constants.POS_INF_NUMERIC_SUMMARY_OP_INDEX]
if nan_count > 0 or neg_inf_count > 0 or pos_inf_count > 0:
return NumericsAlert(
device_name, tensor_name, event.wall_time, nan_count, neg_inf_count,
pos_inf_count)
return None
|