org.apache.flink.configuration.MemorySize Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.stream.IntStream;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.GIGA_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.KILO_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.MEGA_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.TERA_BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.hasUnit;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* MemorySize is a representation of a number of bytes, viewable in different units.
*
* Parsing
*
* The size can be parsed from a text expression. If the expression is a pure number, the value
* will be interpreted as bytes.
*/
@PublicEvolving
public class MemorySize implements java.io.Serializable, Comparable {
private static final long serialVersionUID = 1L;
public static final MemorySize ZERO = new MemorySize(0L);
public static final MemorySize MAX_VALUE = new MemorySize(Long.MAX_VALUE);
private static final List ORDERED_UNITS =
Arrays.asList(BYTES, KILO_BYTES, MEGA_BYTES, GIGA_BYTES, TERA_BYTES);
// ------------------------------------------------------------------------
/** The memory size, in bytes. */
private final long bytes;
/** The memorized value returned by toString(). */
private transient String stringified;
/** The memorized value returned by toHumanReadableString(). */
private transient String humanReadableStr;
/**
* Constructs a new MemorySize.
*
* @param bytes The size, in bytes. Must be zero or larger.
*/
public MemorySize(long bytes) {
checkArgument(bytes >= 0, "bytes must be >= 0");
this.bytes = bytes;
}
public static MemorySize ofMebiBytes(long mebiBytes) {
return new MemorySize(mebiBytes << 20);
}
// ------------------------------------------------------------------------
/** Gets the memory size in bytes. */
public long getBytes() {
return bytes;
}
/** Gets the memory size in Kibibytes (= 1024 bytes). */
public long getKibiBytes() {
return bytes >> 10;
}
/** Gets the memory size in Mebibytes (= 1024 Kibibytes). */
public int getMebiBytes() {
return (int) (bytes >> 20);
}
/** Gets the memory size in Gibibytes (= 1024 Mebibytes). */
public long getGibiBytes() {
return bytes >> 30;
}
/** Gets the memory size in Tebibytes (= 1024 Gibibytes). */
public long getTebiBytes() {
return bytes >> 40;
}
// ------------------------------------------------------------------------
@Override
public int hashCode() {
return (int) (bytes ^ (bytes >>> 32));
}
@Override
public boolean equals(Object obj) {
return obj == this
|| (obj != null
&& obj.getClass() == this.getClass()
&& ((MemorySize) obj).bytes == this.bytes);
}
@Override
public String toString() {
if (stringified == null) {
stringified = formatToString();
}
return stringified;
}
private String formatToString() {
MemoryUnit highestIntegerUnit =
IntStream.range(0, ORDERED_UNITS.size())
.sequential()
.filter(idx -> bytes % ORDERED_UNITS.get(idx).getMultiplier() != 0)
.boxed()
.findFirst()
.map(
idx -> {
if (idx == 0) {
return ORDERED_UNITS.get(0);
} else {
return ORDERED_UNITS.get(idx - 1);
}
})
.orElse(BYTES);
return String.format(
"%d %s",
bytes / highestIntegerUnit.getMultiplier(), highestIntegerUnit.getUnits()[1]);
}
public String toHumanReadableString() {
if (humanReadableStr == null) {
humanReadableStr = formatToHumanReadableString();
}
return humanReadableStr;
}
private String formatToHumanReadableString() {
MemoryUnit highestUnit =
IntStream.range(0, ORDERED_UNITS.size())
.sequential()
.filter(idx -> bytes > ORDERED_UNITS.get(idx).getMultiplier())
.boxed()
.max(Comparator.naturalOrder())
.map(ORDERED_UNITS::get)
.orElse(BYTES);
if (highestUnit == BYTES) {
return String.format("%d %s", bytes, BYTES.getUnits()[1]);
} else {
double approximate = 1.0 * bytes / highestUnit.getMultiplier();
return String.format(
Locale.ROOT,
"%.3f%s (%d bytes)",
approximate,
highestUnit.getUnits()[1],
bytes);
}
}
@Override
public int compareTo(MemorySize that) {
return Long.compare(this.bytes, that.bytes);
}
// ------------------------------------------------------------------------
// Calculations
// ------------------------------------------------------------------------
public MemorySize add(MemorySize that) {
return new MemorySize(Math.addExact(this.bytes, that.bytes));
}
public MemorySize subtract(MemorySize that) {
return new MemorySize(Math.subtractExact(this.bytes, that.bytes));
}
public MemorySize multiply(double multiplier) {
checkArgument(multiplier >= 0, "multiplier must be >= 0");
BigDecimal product =
BigDecimal.valueOf(this.bytes).multiply(BigDecimal.valueOf(multiplier));
if (product.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) {
throw new ArithmeticException("long overflow");
}
return new MemorySize(product.longValue());
}
public MemorySize divide(long by) {
checkArgument(by >= 0, "divisor must be >= 0");
return new MemorySize(bytes / by);
}
// ------------------------------------------------------------------------
// Parsing
// ------------------------------------------------------------------------
/**
* Parses the given string as as MemorySize.
*
* @param text The string to parse
* @return The parsed MemorySize
* @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
*/
public static MemorySize parse(String text) throws IllegalArgumentException {
return new MemorySize(parseBytes(text));
}
/**
* Parses the given string with a default unit.
*
* @param text The string to parse.
* @param defaultUnit specify the default unit.
* @return The parsed MemorySize.
* @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
*/
public static MemorySize parse(String text, MemoryUnit defaultUnit)
throws IllegalArgumentException {
if (!hasUnit(text)) {
return parse(text + defaultUnit.getUnits()[0]);
}
return parse(text);
}
/**
* Parses the given string as bytes. The supported expressions are listed under {@link
* MemorySize}.
*
* @param text The string to parse
* @return The parsed size, in bytes.
* @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
*/
public static long parseBytes(String text) throws IllegalArgumentException {
checkNotNull(text, "text");
final String trimmed = text.trim();
checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
final int len = trimmed.length();
int pos = 0;
char current;
while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') {
pos++;
}
final String number = trimmed.substring(0, pos);
final String unit = trimmed.substring(pos).trim().toLowerCase(Locale.US);
if (number.isEmpty()) {
throw new NumberFormatException("text does not start with a number");
}
final long value;
try {
value = Long.parseLong(number); // this throws a NumberFormatException on overflow
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"The value '"
+ number
+ "' cannot be re represented as 64bit number (numeric overflow).");
}
final long multiplier = parseUnit(unit).map(MemoryUnit::getMultiplier).orElse(1L);
final long result = value * multiplier;
// check for overflow
if (result / multiplier != value) {
throw new IllegalArgumentException(
"The value '"
+ text
+ "' cannot be re represented as 64bit number of bytes (numeric overflow).");
}
return result;
}
private static Optional parseUnit(String unit) {
if (matchesAny(unit, BYTES)) {
return Optional.of(BYTES);
} else if (matchesAny(unit, KILO_BYTES)) {
return Optional.of(KILO_BYTES);
} else if (matchesAny(unit, MEGA_BYTES)) {
return Optional.of(MEGA_BYTES);
} else if (matchesAny(unit, GIGA_BYTES)) {
return Optional.of(GIGA_BYTES);
} else if (matchesAny(unit, TERA_BYTES)) {
return Optional.of(TERA_BYTES);
} else if (!unit.isEmpty()) {
throw new IllegalArgumentException(
"Memory size unit '"
+ unit
+ "' does not match any of the recognized units: "
+ MemoryUnit.getAllUnits());
}
return Optional.empty();
}
private static boolean matchesAny(String str, MemoryUnit unit) {
for (String s : unit.getUnits()) {
if (s.equals(str)) {
return true;
}
}
return false;
}
/**
* Enum which defines memory unit, mostly used to parse value from configuration file.
*
* To make larger values more compact, the common size suffixes are supported:
*
*
* - 1b or 1bytes (bytes)
*
- 1k or 1kb or 1kibibytes (interpreted as kibibytes = 1024 bytes)
*
- 1m or 1mb or 1mebibytes (interpreted as mebibytes = 1024 kibibytes)
*
- 1g or 1gb or 1gibibytes (interpreted as gibibytes = 1024 mebibytes)
*
- 1t or 1tb or 1tebibytes (interpreted as tebibytes = 1024 gibibytes)
*
*/
public enum MemoryUnit {
BYTES(new String[] {"b", "bytes"}, 1L),
KILO_BYTES(new String[] {"k", "kb", "kibibytes"}, 1024L),
MEGA_BYTES(new String[] {"m", "mb", "mebibytes"}, 1024L * 1024L),
GIGA_BYTES(new String[] {"g", "gb", "gibibytes"}, 1024L * 1024L * 1024L),
TERA_BYTES(new String[] {"t", "tb", "tebibytes"}, 1024L * 1024L * 1024L * 1024L);
private final String[] units;
private final long multiplier;
MemoryUnit(String[] units, long multiplier) {
this.units = units;
this.multiplier = multiplier;
}
public String[] getUnits() {
return units;
}
public long getMultiplier() {
return multiplier;
}
public static String getAllUnits() {
return concatenateUnits(
BYTES.getUnits(),
KILO_BYTES.getUnits(),
MEGA_BYTES.getUnits(),
GIGA_BYTES.getUnits(),
TERA_BYTES.getUnits());
}
public static boolean hasUnit(String text) {
checkNotNull(text, "text");
final String trimmed = text.trim();
checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
final int len = trimmed.length();
int pos = 0;
char current;
while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') {
pos++;
}
final String unit = trimmed.substring(pos).trim().toLowerCase(Locale.US);
return unit.length() > 0;
}
private static String concatenateUnits(final String[]... allUnits) {
final StringBuilder builder = new StringBuilder(128);
for (String[] units : allUnits) {
builder.append('(');
for (String unit : units) {
builder.append(unit);
builder.append(" | ");
}
builder.setLength(builder.length() - 3);
builder.append(") / ");
}
builder.setLength(builder.length() - 3);
return builder.toString();
}
}
}