io.bdeploy.bhive.op.remote.FetchOperation Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of api Show documentation
Public API including dependencies, ready to be used for integrations and plugins.
There is a newer version: 7.3.6
package io.bdeploy.bhive.op.remote;

import static io.bdeploy.common.util.RuntimeAssert.assertNotNull;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.bdeploy.bhive.BHive;
import io.bdeploy.bhive.model.Manifest;
import io.bdeploy.bhive.model.Manifest.Key;
import io.bdeploy.bhive.model.ObjectId;
import io.bdeploy.bhive.op.CopyOperation;
import io.bdeploy.bhive.op.ObjectExistsOperation;
import io.bdeploy.bhive.op.ObjectExistsOperation.Result;
import io.bdeploy.bhive.op.ObjectReadOperation;
import io.bdeploy.bhive.remote.RemoteBHive;
import io.bdeploy.common.ActivityReporter.Activity;
import io.bdeploy.common.util.PathHelper;
import jakarta.ws.rs.core.UriBuilder;

/**
 * Fetches manifests from a remote {@link BHive} to the local {@link BHive}. If no
 * manifests are given, all remotely available manifests are fetched.
 */
public class FetchOperation extends TransactedRemoteOperation {

    private static final Logger log = LoggerFactory.getLogger(FetchOperation.class);

    private final SortedSet manifests = new TreeSet<>();
    private String hiveName;

    @Override
    public TransferStatistics callTransacted() throws Exception {
        TransferStatistics stats = new TransferStatistics();
        assertNotNull(getRemote(), "Remote not set");

        SortedSet requiredManifests = new TreeSet<>();
        SortedSet toFetchRootTrees = new TreeSet<>();

        Instant start = Instant.now();
        try (Activity activity = getActivityReporter().start("Fetching", -1)) {
            try (RemoteBHive rh = RemoteBHive.forService(getRemote(), hiveName, getActivityReporter())) {
                // if manifests are empty, the array will be empty, returning all manifests on the remote
                String[] manifestsAsArray = manifests.stream().map(Manifest.Key::toString).toArray(String[]::new);
                SortedMap manifest2Tree = rh.getManifestInventory(manifestsAsArray);
                if (manifests.isEmpty()) {
                    manifests.addAll(manifest2Tree.keySet());
                }

                for (Manifest.Key key : manifests) {
                    if (!manifest2Tree.containsKey(key)) {
                        log.warn("Requested manifest not found anymore on remote: {}", key);
                        continue;
                    }
                    if (getManifestDatabase().hasManifest(key)) {
                        continue;
                    }
                    requiredManifests.add(key);
                    toFetchRootTrees.add(manifest2Tree.get(key));
                }
                if (requiredManifests.isEmpty()) {
                    return stats;
                }

                // STEP 1: Figure out required trees for the roots to fetch
                Set requiredTrees = new LinkedHashSet<>();
                toFetchRootTrees.forEach(t -> requiredTrees.addAll(rh.getRequiredTrees(t)));

                // STEP 2: Figure out which trees we already have locally.
                Result treeResult = execute(new ObjectExistsOperation().addAll(requiredTrees));

                // STEP 3: Find objects for all missing objects, filtering trees we have.
                Set requiredObjects = new LinkedHashSet<>();
                if (!treeResult.missing.isEmpty()) {
                    requiredObjects = rh.getRequiredObjects(treeResult.missing, treeResult.existing);
                }

                // STEP 4: Check which of the required objects we already have
                Result objectResult = execute(new ObjectExistsOperation().addAll(requiredObjects));

                // STEP 5: Fetch from the remote all required objects and manifests.
                TransferStatistics fetchStats = fetch(rh, objectResult.missing, requiredManifests);

                // Update statistics with some new knowledge. the fetch call can only know a few numbers,
                // as for instance the number of trees is irrelevant during actual operation.
                stats.transferSize = fetchStats.transferSize;
                stats.sumManifests = fetchStats.sumManifests;
                stats.sumMissingObjects = fetchStats.sumMissingObjects;
                stats.sumTrees = requiredTrees.size();
                stats.sumMissingTrees = treeResult.missing.size();
            }
        } finally {
            stats.duration = Duration.between(start, Instant.now()).toMillis();
        }
        return stats;
    }

    public FetchOperation addManifest(Manifest.Key key) {
        manifests.add(key);
        return this;
    }

    public FetchOperation addManifest(Collection keys) {
        manifests.addAll(keys);
        return this;
    }

    public FetchOperation setHiveName(String name) {
        hiveName = name;
        return this;
    }

    public SortedSet getManifests() {
        return manifests;
    }

    private TransferStatistics fetch(RemoteBHive rh, Set objects, Set manifests) throws IOException {
        try {
            return fetchAsStream(rh, objects, manifests);
        } catch (UnsupportedOperationException ex) {
            return fetchAsZip(rh, objects, manifests);
        }
    }

    private TransferStatistics fetchAsZip(RemoteBHive rh, Set objects, Set manifests) throws IOException {
        Path z = rh.fetch(objects, manifests);
        try (BHive zHive = new BHive(UriBuilder.fromUri("jar:" + z.toUri()).build(), null, getActivityReporter())) {
            TransferStatistics t = zHive.execute(new CopyOperation().setDestinationHive(this).setPartialAllowed(false));
            t.transferSize = Files.size(z); // transferred size != actual size.
            return t;
        } finally {
            try {
                PathHelper.deleteIfExistsRetry(z);
            } catch (Exception e) {
                log.warn("Cannot clean temporary file while fetching", e);
            }
        }
    }

    private TransferStatistics fetchAsStream(RemoteBHive rh, Set objects, Set manifests) {
        InputStream stream = rh.fetchAsStream(objects, manifests);
        return execute(new ObjectReadOperation().stream(stream));
    }

}