All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.cloudbus.cloudsim.allocationpolicies.migration.VmAllocationPolicyMigrationAbstract Maven / Gradle / Ivy

Go to download

CloudSim Plus: A modern, highly extensible and easier-to-use Java 8 Framework for Modeling and Simulation of Cloud Computing Infrastructures and Services

There is a newer version: 8.0.0
Show newest version
/*
 * Title:        CloudSim Toolkit
 * Description:  CloudSim (Cloud Simulation) Toolkit for Modeling and Simulation of Clouds
 * Licence:      GPL - http://www.gnu.org/copyleft/gpl.html
 *
 * Copyright (c) 2009-2012, The University of Melbourne, Australia
 */
package org.cloudbus.cloudsim.allocationpolicies.migration;

import org.cloudbus.cloudsim.allocationpolicies.VmAllocationPolicy;
import org.cloudbus.cloudsim.allocationpolicies.VmAllocationPolicyAbstract;
import org.cloudbus.cloudsim.core.Simulation;
import org.cloudbus.cloudsim.hosts.Host;
import org.cloudbus.cloudsim.selectionpolicies.power.PowerVmSelectionPolicy;
import org.cloudbus.cloudsim.vms.Vm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static java.util.Comparator.comparingDouble;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;

/**
 * An abstract VM allocation policy that dynamically optimizes the
 * VM allocation (placement) using migration.
 * It's a Best Fit policy which selects the Host with most efficient power usage to place a given VM.
 * Such a behaviour can be overridden by sub-classes.
 *
 * 

If you are using any algorithms, policies or workload included in the * power package please cite the following paper: * *

*

* * @author Anton Beloglazov * @author Manoel Campos da Silva Filho * @since CloudSim Toolkit 3.0 */ public abstract class VmAllocationPolicyMigrationAbstract extends VmAllocationPolicyAbstract implements VmAllocationPolicyMigration { private static final Logger logger = LoggerFactory.getLogger(VmAllocationPolicyMigrationAbstract.class.getSimpleName()); /**@see #getUnderUtilizationThreshold() */ private double underUtilizationThreshold; /** * The vm selection policy. */ private PowerVmSelectionPolicy vmSelectionPolicy; /** * A map between a VM and the host where it is placed. */ private final Map savedAllocation; /** * A map of CPU utilization history (in percentage) for each host, where * each key is a hos and each value is the CPU utilization percentage history. * * @todo there is inconsistency between these data. * Into the Host, it is stored the actual utilization for the given time. * Here it is stored the utilization as it was computed * by the VmAllocationPolicy implementation. * For instance, the {@link VmAllocationPolicyMigrationLocalRegression} * used Local Regression to predict Host utilization * and such value will be stored in this map. * However, these duplicate and inconsistent data * are confusing and error prone. */ private final Map> utilizationHistory; /** * @see #getMetricHistory() */ private final Map> metricHistory; /** * @see #getTimeHistory() */ private final Map> timeHistory; /** * Creates a VmAllocationPolicyMigrationAbstract. * * @param vmSelectionPolicy the policy that defines how VMs are selected for migration */ public VmAllocationPolicyMigrationAbstract(final PowerVmSelectionPolicy vmSelectionPolicy) { this(vmSelectionPolicy, null); } /** * Creates a new VmAllocationPolicy, changing the {@link Function} to select a Host for a Vm. * @param vmSelectionPolicy the policy that defines how VMs are selected for migration * @param findHostForVmFunction a {@link Function} to select a Host for a given Vm. * Passing null makes the Function to be set as the default {@link #findHostForVm(Vm)}. * @see VmAllocationPolicy#setFindHostForVmFunction(java.util.function.BiFunction) */ public VmAllocationPolicyMigrationAbstract( final PowerVmSelectionPolicy vmSelectionPolicy, final BiFunction> findHostForVmFunction) { super(findHostForVmFunction); this.underUtilizationThreshold = 0.35; this.savedAllocation = new HashMap<>(); this.utilizationHistory = new HashMap<>(); this.metricHistory = new HashMap<>(); this.timeHistory = new HashMap<>(); setVmSelectionPolicy(vmSelectionPolicy); } @Override public Map getOptimizedAllocationMap(final List vmList) { //@todo See https://github.com/manoelcampos/cloudsim-plus/issues/94 final Set overloadedHosts = getOverloadedHosts(); printOverUtilizedHosts(overloadedHosts); saveAllocation(); final Map migrationMap = getMigrationMapFromOverloadedHosts(overloadedHosts); updateMigrationMapFromUnderloadedHosts(overloadedHosts, migrationMap); restoreAllocation(); return migrationMap; } /** * Updates the map of VMs that will be migrated from under utilized hosts. * * @param overloadedHosts the List of over utilized hosts * @param migrationMap current migration map that will be updated */ private void updateMigrationMapFromUnderloadedHosts( final Set overloadedHosts, final Map migrationMap) { final List switchedOffHosts = getSwitchedOffHosts(); // overloaded hosts + hosts that are selected to migrate VMs from overloaded hosts final Set ignoredSourceHosts = new HashSet<>(); ignoredSourceHosts.addAll(overloadedHosts); ignoredSourceHosts.addAll(switchedOffHosts); /* During the computation of the new placement for VMs the current VM placement is changed temporarily, before the actual migration of VMs. If VMs are being migrated from overloaded Hosts, they in fact already were removed from such Hosts and moved to destination ones. The target Host that maybe were shut down, might become underloaded too. This way, such Hosts are added to be ignored when looking for underloaded Hosts. See https://github.com/manoelcampos/cloudsim-plus/issues/94 */ ignoredSourceHosts.addAll(migrationMap.values()); // overloaded + underloaded hosts final Set ignoredTargetHosts = new HashSet<>(); ignoredTargetHosts.addAll(overloadedHosts); ignoredTargetHosts.addAll(switchedOffHosts); final int numberOfHosts = getHostList().size(); while (true) { if (numberOfHosts == ignoredSourceHosts.size()) { break; } final Host underloadedHost = getUnderloadedHost(ignoredSourceHosts); if (underloadedHost == Host.NULL) { break; } logger.info("{}: PowerVmAllocationPolicy: Underloaded hosts: {}", getDatacenter().getSimulation().clock(), underloadedHost); ignoredSourceHosts.add(underloadedHost); ignoredTargetHosts.add(underloadedHost); final List vmsToMigrateFromHost = getVmsToMigrateFromUnderUtilizedHost(underloadedHost); if (!vmsToMigrateFromHost.isEmpty()) { if(logger.isInfoEnabled()) { logger.info(" VMs to be reallocated from the underloaded {}: {} {}", underloadedHost, System.lineSeparator(), getVmIds(vmsToMigrateFromHost)); } final Map newVmPlacement = getNewVmPlacementFromUnderloadedHost( vmsToMigrateFromHost, ignoredTargetHosts); ignoredSourceHosts.addAll(extractHostListFromMigrationMap(newVmPlacement)); migrationMap.putAll(newVmPlacement); } } } private String getVmIds(final List vmList) { return vmList.stream().map(vm -> String.valueOf(vm.getId())).collect(Collectors.joining(", ")); } /** * Prints the over utilized hosts. * * @param overloadedHosts the over utilized hosts */ private void printOverUtilizedHosts(final Set overloadedHosts) { if (!overloadedHosts.isEmpty() && logger.isWarnEnabled()) { final String hosts = overloadedHosts.stream().map(this::overloadedHostToString).collect(Collectors.joining(System.lineSeparator())); logger.warn("{}: PowerVmAllocationPolicy: Overloaded hosts in {}{}:{}", getDatacenter().getSimulation().clock(), getDatacenter(), System.lineSeparator(), hosts); } } private String overloadedHostToString(final Host h) { return String.format( " Host %d (upper CPU threshold %.2f, current utilization: %.2f)", h.getId(), getOverUtilizationThreshold(h), h.getUtilizationOfCpu()); } /** * Gets the power consumption different after the supposed placement of a VM into a given Host * and the original Host power consumption. * * @param host the host to check the power consumption * @param vm the candidate vm * @return the host power consumption different after the supposed VM placement or 0 if the power * consumption could not be determined */ protected double getPowerAfterAllocationDifference(final Host host, final Vm vm){ final double powerAfterAllocation = getPowerAfterAllocation(host, vm); if (powerAfterAllocation > 0) { return powerAfterAllocation - host.getPowerModel().getPower(); } return 0; } /** * Checks if a host will be over utilized after placing of a candidate VM. * * @param host the host to verify * @param vm the candidate vm * @return true, if the host will be over utilized after VM placement; * false otherwise */ private boolean isNotHostOverloadedAfterAllocation(final Host host, final Vm vm) { if (!host.createTemporaryVm(vm)) { return false; } final double usagePercent = getHostCpuPercentRequested(host); final boolean isNotHostOverUsedAfterAllocation = !isHostOverloaded(host, usagePercent); host.destroyTemporaryVm(vm); return isNotHostOverUsedAfterAllocation; } /** * {@inheritDoc} * It's based on current CPU usage. * * @param host {@inheritDoc} * @return {@inheritDoc} */ @Override public boolean isHostOverloaded(final Host host) { final double upperThreshold = getOverUtilizationThreshold(host); addHistoryEntryIfAbsent(host, upperThreshold); return isHostOverloaded(host, host.getUtilizationOfCpu()); } /** * Checks if a Host is overloaded based on the given CPU utilization percent. * @param host the Host to check * @param cpuUsagePercent the Host's CPU utilization percent. The values may be: *
    *
  • the current CPU utilization if you want to check if the Host is overloaded right now;
  • *
  • the requested CPU utilization after temporarily placing a VM into the Host * just to check if it supports that VM without being overloaded. * In this case, if the Host doesn't support the already placed temporary VM, * the method will return true to indicate the Host will be overloaded * if the VM is actually placed into it. *
  • *
* @return true if the Host is overloaded, false otherwise */ private boolean isHostOverloaded(final Host host, final double cpuUsagePercent){ final double upperThreshold = getOverUtilizationThreshold(host); addHistoryEntryIfAbsent(host, upperThreshold); return cpuUsagePercent > upperThreshold; } /** * Checks if a host is under utilized, based on current CPU usage. * * @param host the host * @return true, if the host is under utilized; false otherwise */ @Override public boolean isHostUnderloaded(final Host host) { return getHostCpuPercentRequested(host) < getUnderUtilizationThreshold(); } @Override public Optional findHostForVm(final Vm vm) { final Set excludedHosts = new HashSet<>(); excludedHosts.add(vm.getHost()); return findHostForVm(vm, excludedHosts); } /** * Finds a Host that has enough resources to place a given VM and that will not * be overloaded after the placement. The selected Host will be that * one with most efficient power usage for the given VM. * *

This method performs the basic filtering and delegates additional ones * and the final selection of the Host to other method.

* * @param vm the VM * @param excludedHosts the excluded hosts * @return an {@link Optional} containing a suitable Host to place the VM or an empty {@link Optional} if not found * @see #findHostForVmInternal(Vm, Stream) */ public Optional findHostForVm(final Vm vm, final Set excludedHosts) { /*The predicate always returns true to indicate that in fact it is not applying any additional filter.*/ return findHostForVm(vm, excludedHosts, host -> true); } /** * Finds a Host that has enough resources to place a given VM and that will not * be overloaded after the placement. The selected Host will be that * one with most efficient power usage for the given VM. * *

This method performs the basic filtering and delegates additional ones * and the final selection of the Host to other method.

* * @param vm the VM * @param excludedHosts the excluded hosts * @param predicate an additional {@link Predicate} to be used to filter * the Host to place the VM * @return an {@link Optional} containing a suitable Host to place the VM or an empty {@link Optional} if not found * @see #findHostForVmInternal(Vm, Stream) */ public Optional findHostForVm(final Vm vm, final Set excludedHosts, final Predicate predicate) { final Stream stream = this.getHostList().stream() .filter(h -> !excludedHosts.contains(h)) .filter(h -> h.isSuitableForVm(vm)) .filter(h -> isNotHostOverloadedAfterAllocation(h, vm)) .filter(predicate); return findHostForVmInternal(vm, stream); } /** * Applies additional filters to the Hosts Stream and performs the actual Host selection. * This method is a Stream's final operation, that it, it closes the Stream and returns an {@link Optional} value. * *

This method can be overridden by sub-classes to change the method used to select the Host for the given VM.

* * @param vm the VM to find a Host to be placed into * @param hostStream a {@link Stream} containing the Hosts after passing the basic filtering * @return an {@link Optional} containing a suitable Host to place the VM or an empty {@link Optional} if not found * @see #findHostForVm(Vm, Set) * @see #additionalHostFilters(Vm, Stream) */ protected Optional findHostForVmInternal(final Vm vm, final Stream hostStream){ final Comparator hostPowerConsumptionComparator = comparingDouble(h -> getPowerAfterAllocationDifference(h, vm)); return additionalHostFilters(vm, hostStream).min(hostPowerConsumptionComparator); } /** * Applies additional filters to select a Host to place a given VM. * This implementation filters the stream of Hosts to get those ones * that the placement of the VM impacts its power usage. * *

This method can be overridden by sub-classes to change filtering.

* * @param vm the VM to find a Host to be placed into * @param hostStream a {@link Stream} containing the Hosts after passing the basic filtering * @return the Hosts {@link Stream} after applying the additional filters */ private Stream additionalHostFilters(final Vm vm, final Stream hostStream){ return hostStream.filter(h -> getPowerAfterAllocation(h, vm) > 0); } /** * Extracts the host list from a migration map. * * @param migrationMap the migration map * @return the list */ private List extractHostListFromMigrationMap(final Map migrationMap) { return migrationMap.entrySet().stream() .map(Map.Entry::getValue) .collect(toList()); } /** * Gets a new VM placement considering the list of VM to migrate * from overloaded Hosts. * * @param overloadedHosts the list of overloaded Hosts * @return the new VM placement map where each key is a VM * and each value is the Host to place it. */ private Map getMigrationMapFromOverloadedHosts(final Set overloadedHosts) { if(overloadedHosts.isEmpty()) { return new HashMap<>(); } final List vmsToMigrate = getVmsToMigrateFromOverloadedHosts(overloadedHosts); sortByCpuUtilization(vmsToMigrate, getDatacenter().getSimulation().clock()); final Map migrationMap = new HashMap<>(); final StringBuilder sb = new StringBuilder(); for (final Vm vm : vmsToMigrate) { findHostForVm(vm, overloadedHosts).ifPresent(targetHost -> { addVmToMigrationMap(migrationMap, vm, targetHost); appendVmMigrationMsgToStringBuilder(sb, vm, targetHost); }); } logger.info("Reallocation of VMs from overloaded hosts: {}{}", System.lineSeparator(), sb.toString()); return migrationMap; } private void appendVmMigrationMsgToStringBuilder(final StringBuilder sb, final Vm vm, final Host targetHost) { if(logger.isInfoEnabled()) { sb.append(" ").append(vm).append(" will be migrated from ") .append(vm.getHost()).append(" to ").append(targetHost) .append(System.lineSeparator()); } } /** * Gets a new placement for VMs from an underloaded host. * * @param vmsToMigrate the list of VMs to migrate from the underloaded Host * @param excludedHosts the list of hosts that aren't selected as * destination hosts * @return the new vm placement for the given VMs */ private Map getNewVmPlacementFromUnderloadedHost( final List vmsToMigrate, final Set excludedHosts) { final Map migrationMap = new HashMap<>(); sortByCpuUtilization(vmsToMigrate, getDatacenter().getSimulation().clock()); for (final Vm vm : vmsToMigrate) { //try to find a target Host to place a VM from an underloaded Host that is not underloaded too final Optional optional = findHostForVm(vm, excludedHosts, host -> !isHostUnderloaded(host)); if (!optional.isPresent()) { logger.warn( "A new Host, which isn't also underloaded or won't be overloaded, couldn't be found to migrate {}{}.Migration of VMs from the underloaded {} cancelled.", vm, System.lineSeparator(), vm.getHost()); return new HashMap<>(); } addVmToMigrationMap(migrationMap, vm, optional.get()); } return migrationMap; } /** * Sort a given list of VMs by descending order of CPU utilization. * * @param vmList the vm list to be sorted * @param simulationTime the simulation time to get the current CPU utilization for each Vm */ private void sortByCpuUtilization(final List vmList, final double simulationTime) { final Comparator comparator = comparingDouble(vm -> vm.getTotalCpuMipsUsage(simulationTime)); vmList.sort(comparator.reversed()); } private void addVmToMigrationMap(final Map migrationMap, final Vm vm, final T targetHost) { /* Temporarily creates the VM into the target Host so that when the next VM is got to be migrated, if the same Host is selected as destination, the resource to be used by the previous VM will be considered when assessing the suitability of such a Host for the next VM. */ targetHost.createTemporaryVm(vm); migrationMap.put(vm, targetHost); } /** * Gets the VMs to migrate from Hosts. * * @param overloadedHosts the List of overloaded Hosts * @return the VMs to migrate from hosts */ private List getVmsToMigrateFromOverloadedHosts(final Set overloadedHosts) { final List vmsToMigrate = new LinkedList<>(); for (final Host host : overloadedHosts) { vmsToMigrate.addAll(getVmsToMigrateFromOverloadedHost(host)); } return vmsToMigrate; } private List getVmsToMigrateFromOverloadedHost(final Host host) { final List vmsToMigrate = new LinkedList<>(); while (true) { final Vm vm = getVmSelectionPolicy().getVmToMigrate(host); if (Vm.NULL == vm) { break; } vmsToMigrate.add(vm); /*Temporarily destroys the selected VM into the overloaded Host so that the loop gets VMs from such a Host until it is not overloaded anymore.*/ host.destroyTemporaryVm(vm); if (!isHostOverloaded(host)) { break; } } return vmsToMigrate; } /** * Gets the VMs to migrate from under utilized host. * * @param host the host * @return the vms to migrate from under utilized host */ protected List getVmsToMigrateFromUnderUtilizedHost(final Host host) { return host.getVmList().stream() .filter(vm -> !vm.isInMigration()) .collect(Collectors.toCollection(LinkedList::new)); } /** * Gets the switched off hosts. * * @return the switched off hosts */ protected List getSwitchedOffHosts() { return this.getHostList().stream() .filter(host -> !host.isActive() || host.isFailed()) .collect(toList()); } /** * Gets the List of overloaded hosts. * If a Host is overloaded but it has VMs migrating out, * then it's not included in the returned List * because the VMs to be migrated to move the Host from * the overload state already are in migration. * * @return the over utilized hosts */ private Set getOverloadedHosts() { return this.getHostList().stream() .filter(this::isHostOverloaded) .filter(h -> h.getVmsMigratingOut().isEmpty()) .collect(toSet()); } /** * Gets the most underloaded Host. * If a Host is underloaded but it has VMs migrating in, * then it's not included in the returned List * because the VMs to be migrated to move the Host from * the underload state already are in migration to it. * Likewise, if all VMs are migrating out, nothing has to be * done anymore. It just has to wait the VMs to finish * the migration. * * @param excludedHosts the Hosts that have to be ignored when looking for the under utilized Host * @return the most under utilized host or {@link Host#NULL} if no Host is found */ private Host getUnderloadedHost(final Set excludedHosts) { return this.getHostList().stream() .filter(h -> !excludedHosts.contains(h)) .filter(Host::isActive) .filter(this::isHostUnderloaded) .filter(h -> h.getVmsMigratingIn().isEmpty()) .filter(this::notAllVmsAreMigratingOut) .min(comparingDouble(Host::getUtilizationOfCpu)) .orElse(Host.NULL); } private double getHostCpuPercentRequested(final Host host) { return getHostTotalRequestedMips(host) / host.getTotalMipsCapacity(); } /** * Gets the total MIPS that is currently being used by all VMs inside the Host. * @param host * @return */ private double getHostTotalRequestedMips(final Host host) { return host.getVmList().stream() .mapToDouble(Vm::getCurrentRequestedTotalMips) .sum(); } /** * Checks if all VMs of a Host are NOT migrating out. * In this case, the given Host will not be selected as an underloaded Host at the current moment. * That is: not all VMs are migrating out if at least one VM isn't in migration process. * * @param host the host to check * @return true if at least one VM isn't migrating, false if all VMs are migrating */ protected boolean notAllVmsAreMigratingOut(final Host host) { return host.getVmList().stream().anyMatch(vm -> !vm.isInMigration()); } /** * Saves the current map between a VM and the host where it is place. * * @see #savedAllocation */ private void saveAllocation() { savedAllocation.clear(); for (final Host host : getHostList()) { for (final Vm vm : host.getVmList()) { if (!host.getVmsMigratingIn().contains(vm)) { savedAllocation.put(vm, host); } } } } /** * Restore VM allocation from the allocation history. * * @see #savedAllocation */ private void restoreAllocation() { for (final Host host : getHostList()) { host.destroyAllVms(); host.reallocateMigratingInVms(); } for (final Vm vm : savedAllocation.keySet()) { final Host host = savedAllocation.get(vm); if (!host.createTemporaryVm(vm)) { logger.error("Couldn't restore {} on {}", vm, host); return; } } } /** * Gets the power consumption of a host after the supposed placement of a candidate VM. * The VM is not in fact placed at the host. * * @param host the host to check the power consumption * @param vm the candidate vm * * @return the host power consumption after the supposed VM placement or 0 if the power * consumption could not be determined */ protected double getPowerAfterAllocation(final Host host, final Vm vm) { try { return host.getPowerModel().getPower(getMaxUtilizationAfterAllocation(host, vm)); } catch (IllegalArgumentException e) { logger.error("Power consumption for {} could not be determined: {}", host, e.getMessage()); } return 0; } /** * Gets the max power consumption of a host after placement of a candidate * VM. The VM is not in fact placed at the host. We assume that load is * balanced between PEs. The only restriction is: VM's max MIPS < PE's MIPS * * @param host the host * @param vm the vm * * @return the power after allocation */ protected double getMaxUtilizationAfterAllocation(final Host host, final Vm vm) { final double requestedTotalMips = vm.getCurrentRequestedTotalMips(); final double hostUtilizationMips = getUtilizationOfCpuMips(host); final double hostPotentialMipsUse = hostUtilizationMips + requestedTotalMips; return hostPotentialMipsUse / host.getTotalMipsCapacity(); } /** * Gets the utilization of the CPU in MIPS for the current potentially * allocated VMs. * * @param host the host * * @return the utilization of the CPU in MIPS */ protected double getUtilizationOfCpuMips(final Host host) { double hostUtilizationMips = 0; for (final Vm vm2 : host.getVmList()) { if (host.getVmsMigratingIn().contains(vm2)) { // calculate additional potential CPU usage of a migrating in VM hostUtilizationMips += host.getTotalAllocatedMipsForVm(vm2) * 0.9 / 0.1; } hostUtilizationMips += host.getTotalAllocatedMipsForVm(vm2); } return hostUtilizationMips; } /** * Adds an entry for each history map of a host if it doesn't contain * an entry for the current simulation time. * * @param host the host to add metric history entries * @param metric the metric to be added to the metric history map */ protected void addHistoryEntryIfAbsent(final Host host, final double metric) { timeHistory.putIfAbsent(host, new LinkedList<>()); utilizationHistory.putIfAbsent(host, new LinkedList<>()); metricHistory.putIfAbsent(host, new LinkedList<>()); final Simulation simulation = host.getSimulation(); if (!timeHistory.get(host).contains(simulation.clock())) { timeHistory.get(host).add(simulation.clock()); utilizationHistory.get(host).add(host.getUtilizationOfCpu()); metricHistory.get(host).add(metric); } } /** * Sets the vm selection policy. * * @param vmSelectionPolicy the new vm selection policy */ protected final void setVmSelectionPolicy(final PowerVmSelectionPolicy vmSelectionPolicy) { Objects.requireNonNull(vmSelectionPolicy); this.vmSelectionPolicy = vmSelectionPolicy; } /** * Gets the vm selection policy. * * @return the vm selection policy */ protected PowerVmSelectionPolicy getVmSelectionPolicy() { return vmSelectionPolicy; } @Override public Map> getUtilizationHistory() { return Collections.unmodifiableMap(utilizationHistory); } @Override public Map> getMetricHistory() { return Collections.unmodifiableMap(metricHistory); } @Override public Map> getTimeHistory() { return Collections.unmodifiableMap(timeHistory); } @Override public double getUnderUtilizationThreshold() { return underUtilizationThreshold; } @Override public void setUnderUtilizationThreshold(final double underUtilizationThreshold) { this.underUtilizationThreshold = underUtilizationThreshold; } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy