csip.utils.Services Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of csip-core Show documentation
Show all versions of csip-core Show documentation
The Cloud Services Integration Platform is a SoA implementation to offer a Model-as-a-Service framework, Application Programming Interface, deployment infrastructure, and service implementations for environmental modeling.
/*
* $Id: Services.java c4b9d8c126c7 2020-07-28 [email protected] $
*
* This file is part of the Cloud Services Integration Platform (CSIP),
* a Model-as-a-Service framework, API and application suite.
*
* 2012-2019, Olaf David and others, OMSLab, Colorado State University.
*
* OMSLab licenses this file to you under the MIT license.
* See the LICENSE file in the project root for more information.
*/
package csip.utils;
import csip.Config;
import csip.ModelDataService;
import csip.ServiceException;
import csip.SessionLogger;
import java.io.*;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.security.MessageDigest;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Path;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriBuilder;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.glassfish.jersey.media.multipart.BodyPart;
import org.glassfish.jersey.media.multipart.BodyPartEntity;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
/**
* service utilities.
*
* @author Olaf David
*/
public class Services {
public static final String LOCAL_IP_ADDR = getLocalIP();
public static final int ENSEMBLE_THREADS = 10;
/**
* Returns the current local IP address or an empty string in error case /
* when no network connection is up.
*
* @return Returns the current local IP address or an empty string in error
* case.
* @since 0.1.0
*/
private static String getLocalIP() {
String ipOnly = "";
try {
Enumeration nifs = NetworkInterface.getNetworkInterfaces();
if (nifs == null) {
return "";
}
while (nifs.hasMoreElements()) {
NetworkInterface nif = nifs.nextElement();
if (!nif.isLoopback() && nif.isUp() && !nif.isVirtual()) {
Enumeration adrs = nif.getInetAddresses();
while (adrs.hasMoreElements()) {
InetAddress adr = adrs.nextElement();
if (adr != null && !adr.isLoopbackAddress() && (nif.isPointToPoint() || !adr.isLinkLocalAddress())) {
String adrIP = adr.getHostAddress();
String adrName = nif.isPointToPoint() ? adrIP : adr.getCanonicalHostName();
if (!adrName.equals(adrIP)) {
return adrIP;
} else {
ipOnly = adrIP;
}
}
}
}
}
if (ipOnly.length() == 0) {
return null;
}
return ipOnly;
} catch (SocketException ex) {
return null;
}
}
static {
Calendar uuidEpoch = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
uuidEpoch.clear();
uuidEpoch.set(1582, 9, 15, 0, 0, 0); // 9 = October
epochMillis = uuidEpoch.getTime().getTime();
}
static long epochMillis;
//
// private static long getTime(UUID uuid) {
// return (uuid.timestamp() / 10000L) + epochMillis;
// }
// static long getTime(String uuid) {
// UUID u = UUID.fromString(uuid);
// return (u.timestamp() / 10000L) + epochMillis;
// }
static SimpleDateFormat f = new SimpleDateFormat("/dd/HH");
private static synchronized String getPrefix(String uuid) {
UUID u = UUID.fromString(uuid);
long time = (u.timestamp() / 10000L) + epochMillis;
return f.format(new Date(time));
}
// public static synchronized String getPrefix(UUID uuid) {
// SimpleDateFormat f = new SimpleDateFormat("/dd/HH");
// return f.format(new Date(getTime(uuid)));
// }
public static File getResultsDir(String suid) {
return new File(Config.getString(Config.CSIP_RESULTS_DIR) + getPrefix(suid), suid);
}
public static File getWorkDir(String suid) {
return new File(Config.getString(Config.CSIP_WORK_DIR) + getPrefix(suid), suid);
}
/**
* Create a dummy callable
*
* @return a callable that does nothing.
*/
public static Callable dummyCallable() {
return () -> null;
}
public static class FormDataParameter {
String name;
InputStream is;
String filename;
String value;
public FormDataParameter(BodyPart bp) {
FormDataContentDisposition fd = (FormDataContentDisposition) bp.getContentDisposition();
name = fd.getName();
if (fd.getFileName() != null) {
BodyPartEntity bpe = (BodyPartEntity) bp.getEntity();
is = bpe.getInputStream();
filename = fd.getFileName();
} else {
value = bp.getEntityAs(String.class);
}
}
public InputStream getInputStream() {
return is;
}
public boolean isFile() {
return filename != null;
}
public String getValue() {
return value;
}
public String getFilename() {
return filename;
}
public String getName() {
return name;
}
}
/**
* Creates a map of Strings pointing to the input streams fo files
*
* @param b list of BodyPart
* @return The form parameter
*/
public static Map getFormParameter(List b) {
Map m = new HashMap<>();
for (BodyPart bp : b) {
FormDataContentDisposition fd = (FormDataContentDisposition) bp.getContentDisposition();
m.put(fd.getName(), new FormDataParameter(bp));
}
return m;
}
public static String replaceHostinURI(URI uri, String newHost) throws Exception {
int port = Config.getInt("csip.peer.port", 8080);
UriBuilder b = UriBuilder.fromUri(uri);
URI u = b.host(newHost).port(port).build();
return u.toString();
}
public static String getPublicRequestURL(HttpServletRequest httpReq) {
String requrl = httpReq.getRequestURL().toString();
String p = httpReq.getHeader("X-Forwarded-Proto"); // https?
if (p != null) {
UriBuilder b = UriBuilder.fromUri(requrl);
requrl = b.scheme(p).build().toString();
}
String host = toPublicURL(requrl).toString();
return host;
}
public static URI toPublicURL(String u) {
UriBuilder b = UriBuilder.fromUri(u);
String s = Config.getString("csip.public.scheme");
if (s != null) {
b = b.scheme(s);
}
s = Config.getString("csip.public.host");
if (s != null) {
b = b.host(s);
}
s = Config.getString("csip.public.port");
if (s != null) {
b = b.port(Integer.parseInt(s));
}
s = Config.getString("csip.public.contextprefix");
if (s != null) {
//Find the third '/' because http://google.com:3022/ should basically always be 3 slashes in.
String bUrl = b.build().toString();
int indexOfThirdSlash = StringUtils.ordinalIndexOf(bUrl, "/", 3);
//Split the uri on that slash
String hostAndPort = bUrl.substring(0, indexOfThirdSlash);
String remainder = bUrl.substring(indexOfThirdSlash);
//Put the context prefix in the middle.
String newBUrl = hostAndPort + "/" + s + remainder;
//Read back into b as a URI so we don't lose changes.
b = UriBuilder.fromUri(newBUrl);
}
return b.build();
}
/**
* Maps the incoming URL to an internal one.
*
* @param incoming the original, incoming URL.
* @param path the new path (another service)
* @return internal URL
* @throws MalformedURLException when creating a new URL
*/
public static URI toInternalURL(String incoming, String path) throws MalformedURLException {
UriBuilder internal = UriBuilder.fromUri(incoming);
if (path != null) {
internal = internal.replacePath(path);
}
String uri = Config.getString("csip.internal.uri");
if (uri == null) {
return internal.build();
}
URL url = new URL(uri);
if (url.getPort() != -1) {
internal = internal.port(url.getPort());
}
if (url.getHost() != null) {
internal = internal.host(url.getHost());
}
if (url.getProtocol() != null) {
internal = internal.scheme(url.getProtocol());
}
return internal.build();
}
/**
* Copy the formParameter (files) to a directory, if the file is an archive,
* The content will be extracted.
*
* @param log the session logger
* @param dir the directory to copy file to
* @param forms map of FormDataParameter
* @param unpack if the file is a zip file
* @return the extracted files.
* @throws ServiceException try-catch
*/
public static String[] copyAndExtract(SessionLogger log, File dir,
Map forms, boolean unpack) throws ServiceException {
List files = new ArrayList<>();
if (!dir.exists()) {
dir.mkdirs();
}
for (FormDataParameter fd : forms.values()) {
if (!fd.isFile()) {
continue;
}
String name = fd.getFilename();
name = name.replace('\\', '/');
InputStream fis = fd.getInputStream();
String lcName = name.toLowerCase();
// archives
try {
if ((lcName.endsWith(".bz2") || lcName.endsWith(".gz")) && unpack) {
// wrapper (supports single files, as well as tar.gz / tar.bzls
fis = new CompressorStreamFactory().createCompressorInputStream(new BufferedInputStream(fis));
name = removeExt(name);
lcName = name.toLowerCase();
}
if ((lcName.endsWith(".zip") || lcName.endsWith(".tar")) && unpack) {
ArchiveInputStream is = new ArchiveStreamFactory().createArchiveInputStream(new BufferedInputStream(fis));
ArchiveEntry entry = null;
while ((entry = is.getNextEntry()) != null) {
if (is.canReadEntryData(entry)) {
if (entry.isDirectory()) {
new File(dir, entry.getName()).mkdirs();
} else {
File f = new File(dir, entry.getName());
if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs();
}
try (FileOutputStream ous = new FileOutputStream(f)) {
IOUtils.copy(is, ous);
}
files.add(entry.getName());
f.setLastModified(entry.getLastModifiedDate().getTime());
log.info("Extracted :" + entry.getName() + " as " + f);
}
}
}
is.close();
} else {
File f = new File(dir, name);
if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs();
}
try (FileOutputStream ous = new FileOutputStream(f)) {
IOUtils.copy(fis, ous);
}
files.add(name);
if (log.isLoggable(Level.INFO)) {
log.info("copy form data file: " + name + " to " + dir);
}
}
} catch (CompressorException | ArchiveException | IOException ex) {
throw new ServiceException(ex);
}
try {
fis.close();
} catch (IOException ex) {
log.log(Level.SEVERE, "", ex);
}
}
return files.toArray(new String[files.size()]);
}
public static String removeExt(String name) {
return name.substring(0, name.lastIndexOf("."));
}
public static String removeFirstLastChar(String text) {
return text.substring(1, text.length() - 1);
}
public static String md5(File file) {
if (!file.exists()) {
throw new IllegalArgumentException("not found: " + file);
}
if (file.length() == 0) {
throw new IllegalArgumentException("empty file: " + file);
}
FileInputStream fIn = null;
try {
fIn = new FileInputStream(file);
MessageDigest md = MessageDigest.getInstance("MD5");
FileChannel fChan = fIn.getChannel();
ByteBuffer mBuf = ByteBuffer.allocate((int) fChan.size());
fChan.read(mBuf);
return toHex(md.digest(mBuf.array()));
} catch (Exception ex) {
ex.printStackTrace(System.err);
} finally {
try {
if (fIn != null) {
fIn.close();
}
} catch (IOException ex) {
}
}
return "";
}
//////////////////////////////
// private
/////////////////////////////
private static String toHex(byte[] data) {
StringBuilder buf = new StringBuilder();
for (int i = 0; i < data.length; i++) {
int halfbyte = (data[i] >>> 4) & 0x0F;
int two_halfs = 0;
do {
if ((0 <= halfbyte) && (halfbyte <= 9)) {
buf.append((char) ('0' + halfbyte));
} else {
buf.append((char) ('a' + (halfbyte - 10)));
}
halfbyte = data[i] & 0x0F;
} while (two_halfs++ < 1);
}
return buf.toString();
}
public interface CallableFactory {
Callable create(int i);
}
// public static List> run(int max, final CallableFactory callable) {
// return run(max, Config.getInt("codebase.threadpool", ENSEMBLE_THREADS), callable);
// }
//
//
// public static List> run(int max, int threads, final CallableFactory callable) {
// final CountDownLatch latch = new CountDownLatch(max);
// final int attempts = 3;
// final boolean fail_all = true;
//
// final ExecutorService executor = Executors.newFixedThreadPool(threads);
// List> resp = new ArrayList<>();
// for (int i = 0; i < max; i++) {
// final int ii = i;
// resp.add(executor.submit(new Callable() {
// @Override
// public JSONObject call() throws Exception {
// JSONObject o = null;
// String err_msg = null;
// Callable ca = callable.create(ii);
// int a = attempts;
// // allow trying this multiple times.
// while (a-- > 0 && o == null) {
// try {
// o = ca.call();
// } catch (Exception E) {
// err_msg = E.getMessage();
// }
// }
// if (o == null && fail_all) {
// executor.shutdown();
// throw new ServiceException("Failed service :" + ii + " " + err_msg);
// }
// latch.countDown();
// return o == null ? JSONUtils.error(err_msg) : o;
// }
// }));
// }
// try {
// latch.await();
// } catch (InterruptedException ex) {
// }
// executor.shutdown();
// return resp;
// }
static synchronized ExecutorService getES(int nthreads, int bq_len) {
BlockingQueue bq = new ArrayBlockingQueue<>(nthreads + bq_len);
RejectedExecutionHandler eh = new ThreadPoolExecutor.CallerRunsPolicy();
ExecutorService es = new ThreadPoolExecutor(nthreads, nthreads, 0L, TimeUnit.MILLISECONDS, bq, eh);
return es;
}
public static void runParallel(int count, CallableFactory factory) {
runParallel(count, Config.getInt("csip.service.peers", 4), factory);
}
public static void runParallel(int count, int threads, CallableFactory factory) {
runParallel(count, threads, Config.getInt("csip.internal.call.attempts", 4),
Config.getInt("csip.internal.bq", 4), factory);
}
public static void runParallel(int count, int threads, final int attempts, int bq, CallableFactory factory) {
// have the number of threads being bound by count
int threads_ = Math.min(count, threads);
final ExecutorService exec = getES(threads_, bq);
final CountDownLatch latch = new CountDownLatch(count);
for (int i = 0; i < count; i++) {
final Callable c = factory.create(i);
exec.submit(new Runnable() {
@Override
public void run() {
int a = attempts;
Exception Ex = null;
while (a > 0) {
try {
c.call();
break;
} catch (Exception E) {
System.err.println("Failed #" + a);
Ex = E;
a--;
}
}
if (Ex != null) {
System.err.println("Failed all attempts, last exception:");
Ex.printStackTrace(System.err);
exec.shutdownNow();
}
latch.countDown();
}
});
}
try {
latch.await();
} catch (InterruptedException ex) {
}
exec.shutdownNow();
}
// public static List> run0(int max, int threads, final CallableFactory callable) {
// final CountDownLatch latch = new CountDownLatch(max);
// final ExecutorService executor = Executors.newFixedThreadPool(threads);
// List> resp = new ArrayList<>();
// for (int i = 0; i < max; i++) {
// final int ii = i;
// resp.add(executor.submit(new Callable() {
// @Override
// public JSONObject call() throws Exception {
// JSONObject o = null;
// String err_msg = null;
// Callable ca = callable.create(ii);
// try {
// o = ca.call();
// } catch (Exception E) {
// err_msg = E.getMessage();
// } catch (AssertionError E) {
// err_msg = E.getMessage();
// }
// latch.countDown();
// return (err_msg != null) ? JSONUtils.error(err_msg) : o;
// }
// }));
// }
// try {
// latch.await();
// } catch (InterruptedException ex) {
// }
// executor.shutdownNow();
// return resp;
// }
/**
* run all models at once.
*
* @param models list of callable JSONObject
* @return the list of futures
*/
public static List> runEnsemble(List> models) {
final ExecutorService executor = Executors.newFixedThreadPool(Config.getInt("codebase.threadpool", ENSEMBLE_THREADS));
final CountDownLatch barrier = new CountDownLatch(models.size());
final List> results = new ArrayList<>();
// Model callables
for (final Callable ca : models) {
results.add(executor.submit(new Callable() {
@Override
public JSONObject call() {
JSONObject res = null;
try {
res = ca.call();
} catch (Exception E) {
executor.shutdownNow();
}
barrier.countDown();
return res;
}
}));
}
try {
barrier.await();
} catch (InterruptedException E) {
}
executor.shutdown();
return results;
}
/**
* Slice a original request into single runs.
*
* @param req JSON request
* @param path the path
* @return the mapped list of ensembles.
* @throws JSONException when dealing with JSONObject
*/
public static List> mapEnsemble(JSONObject req, String path) throws JSONException {
String codebase = Config.getString("codebase.url", "http://csip.engr.colostate.edu:8081/rest");
JSONObject metainfo = req.getJSONObject(ModelDataService.KEY_METAINFO);
if (!req.has(ModelDataService.KEY_METAINFO) || !metainfo.has(ModelDataService.KEY_PARAMETERSETS)) {
return null;
}
List> runs = new ArrayList<>();
if (metainfo.has(ModelDataService.KEY_PARAMETERSETS)) {
JSONArray psets = req.getJSONArray(ModelDataService.KEY_PARAMETER);
for (int i = 0; i < metainfo.getInt(ModelDataService.KEY_PARAMETERSETS); i++) {
JSONArray pset = psets.getJSONArray(i);
JSONObject single_req = JSONUtils.newRequest(pset, new JSONObject());
RestCallable mv = new RestCallable(single_req, codebase + path);
runs.add(mv);
}
}
return runs;
}
static boolean isFailed(JSONObject res) throws JSONException {
return res.getJSONObject(ModelDataService.KEY_METAINFO).getString(ModelDataService.KEY_STATUS).equals("Failed");
}
public static JSONObject reduceEnsemble(List> ens, JSONObject orig_req) throws Exception {
JSONArray results = new JSONArray();
for (Future future : ens) {
JSONObject res = future.get();
if (isFailed(res)) {
orig_req.getJSONObject(ModelDataService.KEY_METAINFO).put(ModelDataService.KEY_STATUS, "Failed");
}
results.put(res.get(ModelDataService.KEY_RESULT));
}
return JSONUtils.newResponse(orig_req.getJSONArray(ModelDataService.KEY_PARAMETER), results, orig_req.getJSONObject(ModelDataService.KEY_METAINFO));
}
static public class RestCallable implements Callable {
JSONObject req;
String url;
public RestCallable(JSONObject req, String url) {
this.req = req;
this.url = url;
}
@Override
public JSONObject call() throws Exception {
Client client = ClientBuilder.newClient();
WebTarget service = client.target(UriBuilder.fromUri(url).build());
return service.request(MediaType.APPLICATION_JSON).post(Entity.json(req), JSONObject.class);
}
}
/**
* Check if a class is a CSIP service. (formData/d/or MDS subclass)
*
* @param s the class to check,
* @return if it a CSIP service
*/
public static boolean isCsipService(Class s) {
// Must be ModelDataservice sub class.
if (!ModelDataService.class.isAssignableFrom(s)) {
return false;
}
// check service path, must start with 'formData' or 'd' or 'p'
Path p = s.getAnnotation(Path.class);
if (p == null) {
return false;
}
if (p.value().startsWith("m/") // model service
|| p.value().startsWith("p/") // platform service
|| p.value().startsWith("d/")) { // data service
return true;
}
return false;
}
public static String[] getURIParts(String uri) throws URISyntaxException {
URI b = new URI(uri);
String[] path = b.getPath().split("/");
String[] p = new String[2 + path.length];
p[0] = b.getScheme() + "://";
p[1] = b.getHost();
p[2] = b.getPort() == -1 ? "" : (":" + Integer.toString(b.getPort()));
for (int i = 1; i < path.length; i++) {
p[i + 2] = path[i];
}
return p;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy