Viewing file:  compressor.py (5.12 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Package for histogram compression."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
import numpy as np
# Normal CDF for std_devs: (-Inf, -1.5, -1, -0.5, 0, 0.5, 1, 1.5, Inf)
# naturally gives bands around median of width 1 std dev, 2 std dev, 3 std dev,
# and then the long tail.
NORMAL_HISTOGRAM_BPS = (0, 668, 1587, 3085, 5000, 6915, 8413, 9332, 10000)
CompressedHistogramValue = collections.namedtuple('CompressedHistogramValue',
['basis_point', 'value'])
# TODO(@jart): Unfork these methods.
def compress_histogram_proto(histo, bps=NORMAL_HISTOGRAM_BPS):
"""Creates fixed size histogram by adding compression to accumulated state.
This routine transforms a histogram at a particular step by interpolating its
variable number of buckets to represent their cumulative weight at a constant
number of compression points. This significantly reduces the size of the
histogram and makes it suitable for a two-dimensional area plot where the
output of this routine constitutes the ranges for a single x coordinate.
Args:
histo: A HistogramProto object.
bps: Compression points represented in basis points, 1/100ths of a percent.
Defaults to normal distribution.
Returns:
List of values for each basis point.
"""
# See also: Histogram::Percentile() in core/lib/histogram/histogram.cc
if not histo.num:
return [CompressedHistogramValue(b, 0.0) for b in bps]
bucket = np.array(histo.bucket)
bucket_limit = list(histo.bucket_limit)
weights = (bucket * bps[-1] / (bucket.sum() or 1.0)).cumsum()
values = []
j = 0
while j < len(bps):
i = np.searchsorted(weights, bps[j], side='right')
while i < len(weights):
cumsum = weights[i]
cumsum_prev = weights[i - 1] if i > 0 else 0.0
if cumsum == cumsum_prev: # prevent lerp divide by zero
i += 1
continue
if not i or not cumsum_prev:
lhs = histo.min
else:
lhs = max(bucket_limit[i - 1], histo.min)
rhs = min(bucket_limit[i], histo.max)
weight = _lerp(bps[j], cumsum_prev, cumsum, lhs, rhs)
values.append(CompressedHistogramValue(bps[j], weight))
j += 1
break
else:
break
while j < len(bps):
values.append(CompressedHistogramValue(bps[j], histo.max))
j += 1
return values
def compress_histogram(buckets, bps=NORMAL_HISTOGRAM_BPS):
"""Creates fixed size histogram by adding compression to accumulated state.
This routine transforms a histogram at a particular step by linearly
interpolating its variable number of buckets to represent their cumulative
weight at a constant number of compression points. This significantly reduces
the size of the histogram and makes it suitable for a two-dimensional area
plot where the output of this routine constitutes the ranges for a single x
coordinate.
Args:
buckets: A list of buckets, each of which is a 3-tuple of the form
`(min, max, count)`.
bps: Compression points represented in basis points, 1/100ths of a percent.
Defaults to normal distribution.
Returns:
List of values for each basis point.
"""
# See also: Histogram::Percentile() in core/lib/histogram/histogram.cc
buckets = np.array(buckets)
if not buckets.size:
return [CompressedHistogramValue(b, 0.0) for b in bps]
(minmin, maxmax) = (buckets[0][0], buckets[-1][1])
counts = buckets[:, 2]
right_edges = list(buckets[:, 1])
weights = (counts * bps[-1] / (counts.sum() or 1.0)).cumsum()
result = []
bp_index = 0
while bp_index < len(bps):
i = np.searchsorted(weights, bps[bp_index], side='right')
while i < len(weights):
cumsum = weights[i]
cumsum_prev = weights[i - 1] if i > 0 else 0.0
if cumsum == cumsum_prev: # prevent division-by-zero in `_lerp`
i += 1
continue
if not i or not cumsum_prev:
lhs = minmin
else:
lhs = max(right_edges[i - 1], minmin)
rhs = min(right_edges[i], maxmax)
weight = _lerp(bps[bp_index], cumsum_prev, cumsum, lhs, rhs)
result.append(CompressedHistogramValue(bps[bp_index], weight))
bp_index += 1
break
else:
break
while bp_index < len(bps):
result.append(CompressedHistogramValue(bps[bp_index], maxmax))
bp_index += 1
return result
def _lerp(x, x0, x1, y0, y1):
"""Affinely map from [x0, x1] onto [y0, y1]."""
return y0 + (x - x0) * float(y1 - y0) / (x1 - x0)
|