Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
un paging of calls that require paging. if the api enforces a max of at most e.g. 240, calls in this utility will accept any max, and do paging implicitly
less arguments. Some of the Rest service interface want arguments like request and response object which should at the client side simply remain null (btw I think there are no much of that kind of methods left
exception handling
Parsing of input stream if that it the return value (huge results like {@link nl.vpro.api.rs.v3.media.MediaRestService#changes(String, String, Long, String, String, Integer, Boolean, Deletes, Tail, String)} and {@link nl.vpro.api.rs.v3.media.MediaRestService#iterate(MediaForm, String, String, Long, Integer)} have that.
* @author Michiel Meeuwissen
* @see NpoApiPageUtil
* @see NpoApiImageUtil
*/
@Named
@Slf4j
public class NpoApiMediaUtil implements MediaProvider {
protected static ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(1);
final NpoApiClients clients;
final NpoApiRateLimiter limiter;
// TODO arrange caching via ehcache (ehcache4guice or something)
private int cacheSize = 500;
private Duration cacheTTL = Duration.ofMinutes(5);
private boolean iterateLogProgress = true;
LoadingCache> cache = buildCache();
private static long maxWindow = 10000;
private Instant loggedAboutConnect = Instant.EPOCH;
@Inject
public NpoApiMediaUtil(@NotNull NpoApiClients clients, @NotNull NpoApiRateLimiter limiter) {
this.clients = clients;
this.limiter = limiter;
}
@lombok.Builder
private NpoApiMediaUtil(
@NotNull NpoApiClients clients,
@Nullable NpoApiRateLimiter limiter,
boolean iterateLogProgress,
int cacheSize,
Duration cacheTTL
) {
this.clients = clients;
this.limiter = limiter == null ? new NpoApiRateLimiter() : limiter;
this.cacheTTL = cacheTTL;
this.cacheSize = cacheSize;
this.iterateLogProgress = iterateLogProgress;
}
public NpoApiMediaUtil(NpoApiClients clients) {
this(clients, new NpoApiRateLimiter());
}
public void clearCache() {
cache.invalidateAll();
clients.clearBrowserCache();
}
@Named("npo-api-mediautil.cachesize")
public void setCacheSize(int size) {
cacheSize = size;
cache = buildCache();
}
@Named("npo-api-mediautil.cacheExpiry")
public void setCacheExpiry(String ttl) {
this.cacheTTL = TimeUtils.parseDuration(ttl).orElse(Duration.ofMinutes(5));
cache = buildCache();
}
private LoadingCache> buildCache() {
return CacheBuilder.newBuilder()
.concurrencyLevel(4)
.maximumSize(cacheSize)
.expireAfterWrite(cacheTTL.toMillis(), TimeUnit.MILLISECONDS)
.build(
new CacheLoader>() {
@Override
public @NonNull Optional load(@NonNull String mid) {
limiter.acquire();
try {
MediaObject object = MediaRestClientUtils.loadOrNull(clients.getMediaService(), mid);
limiter.upRate();
//return Optional.ofNullable(object);
return Optional.ofNullable(object);
} catch (RuntimeException se) {
limiter.downRate();
throw se;
}
}
});
}
@SuppressWarnings("unchecked")
public T loadOrNull(String id) throws IOException {
try {
return (T) cache.get(id).orElse(null);
} catch (ExecutionException | UncheckedExecutionException e) {
if (e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
}
if (e.getCause() instanceof RuntimeException) {
throw (RuntimeException) e.getCause();
}
throw new RuntimeException(e);
}
}
public void invalidateCache() {
cache.invalidateAll();
}
public MediaResult listDescendants(String mid, Order order) {
limiter.acquire();
try {
MediaResult result = clients.getMediaService().listDescendants(mid, null, null, order.toString(), 0L, 200);
limiter.upRate();
return result;
} catch (Exception e) {
limiter.downRate();
throw e;
}
}
/**
* Given an api with offset/max formalism, unpage this. The 'maximal' match of NPO API is
* 240, this allows for large max sizes.
*/
public MediaResult unPage(
BiFunction supplier,
Predicate filter,
int max) {
limiter.acquire();
if (filter == null) {
filter = mediaObject -> true;
}
try {
List result = new ArrayList<>();
long offset = 0L;
int batch = 50;
long total;
long found = 0;
do {
MediaResult list = supplier.apply(batch, offset);
total = list.getTotal();
if (list.getSize() == 0) {
break;
}
list.getItems().stream()
.filter(filter).forEach(o -> {
if (result.size() < max) {
result.add(o);
}
});
offset += batch;
found += list.getSize();
if (offset > maxWindow - batch) {
log.info("Offset is getting to big. Breaking");
break;
}
} while (found < total && result.size() < max);
limiter.upRate();
return new MediaResult(result, 0L, max, equalsTo(total));
} catch (Exception e) {
limiter.downRate();
throw e;
}
}
/**
* The api has limits on max size, requiring you to use paging when you want more.
* This method arranges that.
*/
public ProgramResult unPageProgramResult(BiFunction supplier, Predicate filter, int max) {
limiter.acquire();
if (filter == null) {
filter = mediaObject -> true;
}
try {
List result = new ArrayList<>();
long offset = 0L;
int batch = 50;
long total;
long found = 0;
do {
ProgramResult list = supplier.apply(batch, offset);
total = list.getTotal();
if (list.getSize() == 0) {
break;
}
list.getItems().stream().filter(filter).forEach(o -> {
if (result.size() < max) {
result.add(o);
}
});
offset += batch;
found += list.getSize();
if (offset > maxWindow - batch) {
log.info("Offset is getting to big. Breaking");
break;
}
} while (found < total && result.size() < max);
limiter.upRate();
return new ProgramResult(result, 0L, max, equalsTo(total));
} catch (Exception e) {
limiter.downRate();
throw e;
}
}
/**
* Wraps {@link nl.vpro.api.rs.v3.media.MediaRestService#listDescendants(String, String, String, String, Long, Integer)}, but with less arguments.
* @param max The max number of results you want. If this is bigger than the maximum accepted by the API, implicit paging will happen.
*/
public MediaResult listDescendants(String mid, Order order, Predicate filter, int max) {
BiFunction descendants = (batch, offset) ->
clients.getMediaService().listDescendants(mid, null, null, order.toString(), offset, batch);
return unPage(descendants, filter, max);
}
public MediaResult listMembers(String mid, Order order, Predicate filter, int max) {
BiFunction members =
(batch, offset) -> clients.getMediaService().listMembers(mid, null, null,
order.toString(), offset, batch);
return unPage(members, filter, max);
}
public ProgramResult listEpisodes(String mid, Order order, Predicate filter, int max) {
BiFunction members = (batch, offset) -> clients.getMediaService().listEpisodes(mid, null,null, order.toString(), offset, batch);
return unPageProgramResult(members, filter, max);
}
@SuppressWarnings({"unchecked", "OptionalAssignedToNull"})
public MediaObject[] load(String... id) throws IOException {
Optional[] result = new Optional[id.length];
Set toRequest = new LinkedHashSet<>();
for (int i = 0; i < id.length; i++) {
result[i] = cache.getIfPresent(id[i]);
if (result[i] == null) {
toRequest.add(id[i]);
} else {
log.debug("Using {} from cache", id[i]);
}
}
if (!toRequest.isEmpty()) {
limiter.acquire();
try {
String[] array = toRequest.toArray(new String[0]);
MediaObject[] requested = MediaRestClientUtils.load(clients.getMediaService(), array);
for (int j = 0 ; j < array.length; j++) {
Optional optional = Optional.ofNullable(requested[j]);
cache.put(array[j], optional);
for (int i = 0; i < id.length; i++) {
if (id[i].equals(array[j])) {
result[i] = optional;
}
}
}
limiter.upRate();
} catch (ProcessingException pe) {
limiter.downRate();
unwrapIO(pe);
throw pe;
} catch (RuntimeException rte) {
limiter.downRate();
throw rte;
}
}
MediaObject[] resultArray = new MediaObject[id.length];
for (int i = 0; i < id.length; i++) {
resultArray[i] = result[i].orElse(null);
}
return resultArray;
}
public CountedIterator changes(String profile, Instant since, Order order, Integer max) {
return changes(profile, since, null, order, max);
}
public CountedIterator changes(String profile, Instant since, String mid, Order order, Integer max) {
return changes(changesParameters()
.profile(profile)
.since(since)
.mid(mid)
.order(order)
.max(max)
.deletes(Deletes.ID_ONLY)
.tail(Tail.IF_EMPTY)
.build());
}
@Deprecated
public CountedIterator changes(String profile, Boolean profileCheck, Instant since, String mid, Order order, Integer max, Deletes deletes) {
return changes(profile, profileCheck, since, mid, order, max, deletes, Tail.IF_EMPTY);
}
public CountedIterator changes(String profile, Instant since, String mid, Order order, Integer max, Deletes deletes) {
return changes(profile, null, since, mid, order, max, deletes, Tail.IF_EMPTY);
}
public CountedIterator changes(String profile, Instant since) {
return changes(changesParameters().profile(profile).since(since).build());
}
public CountedIterator changes(Instant since) {
return changes(MediaSince.of(since));
}
public CountedIterator changes(MediaSince since) {
return changes(changesParameters().mediaSince(since));
}
@Deprecated
public CountedIterator changes(String profile, Boolean profileCheck, Instant since, String mid, Order order, Integer max, Deletes deletes, Tail tail) {
return changes(changesParameters()
.profile(profile)
.since(since)
.mid(mid)
.order(order)
.max(max)
.deletes(deletes)
.tail(tail)
);
}
public CountedIterator changes(String profile, Instant since, String mid, Order order, Integer max, Deletes deletes, Tail tail) {
return changes(changesParameters()
.profile(profile)
.since(since)
.mid(mid)
.order(order)
.max(max)
.deletes(deletes)
.tail(tail)
);
}
@Getter
@Setter
private Duration waitBetweenChangeListening = Duration.ofSeconds(2);
public Future subscribeToChanges(String profile, Instant since, BooleanSupplier doWhile, final Consumer listener) {
return subscribeToChanges(profile, since, Deletes.ID_ONLY, doWhile, listener);
}
public Future subscribeToChanges(Instant since, BooleanSupplier doWhile, final Consumer listener) {
return subscribeToChanges(null, since, doWhile, listener);
}
public Future subscribeToChanges(
@Nullable String profile,
final Instant initialSince,
Deletes deletes,
BooleanSupplier doWhile,
final Consumer listener) {
return subscribeToChanges(
ChangesFeedParameters.changesParameters()
.profile(profile)
.since(initialSince)
.deletes(deletes)
.build(),
doWhile,
Consumers.ignoreArg1(listener)
).thenApply(MediaSince::getInstant);
}
public CompletableFuture subscribeToChanges(
final ChangesFeedParameters parameters,
final BooleanSupplier doWhile,
final BiConsumer listener) {
if (doWhile.getAsBoolean()) {
return CompletableFuture.supplyAsync(() -> {
ChangesFeedParameters effectiveParameters = parameters;
MediaSince currentSince = parameters.getMediaSince();
while (doWhile.getAsBoolean() && !Thread.currentThread().isInterrupted()) {
try (CountedIterator changes = changes(effectiveParameters)) {
while (changes.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
log.info("Breaking with remaining changes because thread {} is marked interrupted", Thread.currentThread());
break;
}
MediaChange change = changes.next();
currentSince = change.asSince();
listener.accept(currentSince, change);
effectiveParameters = parameters.withMediaSince(currentSince);
}
} catch (NullPointerException npe) {
log.error(npe.getClass().getSimpleName(), npe);
} catch (ConnectException ce) {
if (loggedAboutConnect.isBefore(Instant.now().minus(Duration.ofMinutes(5)))) {
log.info(ce.getClass() + ":" + ce.getMessage());
loggedAboutConnect = Instant.now();
} else {
log.debug(ce.getClass() + ":" + ce.getMessage());
}
} catch (Exception e) {
log.info(e.getClass() + ":" + e.getMessage());
}
try {
synchronized (listener) {
listener.wait(waitBetweenChangeListening.toMillis());
}
} catch (InterruptedException iae) {
log.info("Interrupted");
Thread.currentThread().interrupt();
}
}
log.info("Ready listening for changes until: {}, interrupted: {}", doWhile.getAsBoolean(), Thread.currentThread().isInterrupted());
synchronized (listener) {
listener.notifyAll();
}
return currentSince;
}, EXECUTOR_SERVICE);
} else {
log.info("No started changes listening, because doWhile condition is already false");
return CompletableFuture.completedFuture(parameters.getMediaSince());
}
}
public RedirectList redirects() {
try (Response response = clients.getMediaService().redirects(null)) {
return response.readEntity(RedirectList.class);
}
}
public JsonArrayIterator changes(ChangesFeedParameters.Builder parameters) {
return changes(parameters.build());
}
public JsonArrayIterator changes(ChangesFeedParameters parameters) {
limiter.acquire();
try {
try {
JsonArrayIterator result = MediaRestClientUtils.changes(clients.getMediaServiceNoTimeout(), parameters);
limiter.upRate();
return result;
} catch (ConnectException ce) {
throw ce;
}
} catch (IOException e) {
limiter.downRate();
throw new RuntimeException(clients + ":" + e.getMessage(), e);
}
}
@Deprecated
public JsonArrayIterator changes(String profile, Long since, Order order, Integer max) {
limiter.acquire();
try {
JsonArrayIterator result = MediaRestClientUtils.changes(clients.getMediaServiceNoTimeout(), profile, since, order, max);
limiter.upRate();
return result;
} catch (IOException e) {
limiter.downRate();
throw new RuntimeException(clients + ":" + e.getMessage(), e);
}
}
/**
* Calls {@link nl.vpro.api.rs.v3.media.MediaRestService#iterate(MediaForm, String, String, Long, Integer)}, and wraps the resulting {@link java.io.InputStream} in an {@link Iterator} of {@link MediaObject}}
*/
public CloseableIterator iterate(MediaForm form, String profile) {
limiter.acquire();
try {
CloseableIterator result = MediaRestClientUtils
.iterate(clients.getMediaServiceNoTimeout(), form, profile, iterateLogProgress);
limiter.upRate();
return result;
} catch (Throwable e) {
limiter.downRate();
throw new RuntimeException(clients + ":" + e.getMessage(), e);
}
}
@Override
public T findByMid(boolean loadDeleted, String mid) {
try {
if (loadDeleted) {
throw new UnsupportedOperationException();
}
return (T) load(mid)[0];
} catch (IOException e) {
return null;
}
}
/**
* It is (currently?) not possible to load deleted objects from the frontend api, so
* this defaults to findByMid(false, mid)}
*/
@Override
public T findByMid(String mid) {
return findByMid(false, mid);
}
public MediaType getType(String mid) throws IOException {
MediaObject object = load(mid)[0];
return object == null ? MediaType.MEDIA : object.getMediaType();
}
@Override
public String toString() {
return String.valueOf(clients);
}
public NpoApiClients getClients() {
return clients;
}
public boolean isAvailable() {
return clients.isAvailable();
}
}
