com.metaeffekt.mirror.Mirror Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.mirror;

import com.metaeffekt.artifact.analysis.utils.StringUtils;
import com.metaeffekt.artifact.analysis.utils.TimeUtils;
import com.metaeffekt.artifact.enrichment.InventoryEnricher;
import com.metaeffekt.mirror.download.Download;
import com.metaeffekt.mirror.download.documentation.MirrorMetadata;
import org.metaeffekt.core.inventory.processor.model.AbstractModelBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

import static com.metaeffekt.mirror.Mirror.InfoFileAttributes.LOCK_LOCKED_FILE;
import static com.metaeffekt.mirror.Mirror.InfoFileAttributes.LOCK_LOCKED_SINCE;

public abstract class Mirror {

    private final static Logger LOG = LoggerFactory.getLogger(Mirror.class);

    protected final File baseMirrorDirectory;
    protected final String mirrorIdentifier;
    protected final PropertyFileAccess propertyFiles = new PropertyFileAccess("mirror");
    private final List lockedFiles = new ArrayList<>();

    protected Mirror(File baseMirrorDirectory, String mirrorIdentifier) {
        this.baseMirrorDirectory = baseMirrorDirectory;
        this.mirrorIdentifier = mirrorIdentifier;

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            for (int i = lockedFiles.size() - 1; i >= 0; i--) {
                unlockFile(new File(lockedFiles.get(i)));
            }
        }));

        if (this.getClass().isAnnotationPresent(MirrorMetadata.class)) {
            final MirrorMetadata annotation = this.getClass().getAnnotation(MirrorMetadata.class);
            final boolean isDeprecated = annotation.deprecated();
            final String dirName = annotation.directoryName();
            final String mavenName = annotation.mavenPropertyName();
            if (isDeprecated) {
                LOG.warn("The mirror [{}] [{}] [{}] is deprecated.", this.getClass().getSimpleName(), dirName, mavenName);
            }
        }
    }

    /**
     * If lastModifiedTimestamp is smaller or equal to 0, returns true,
     * currentTime - lastModifiedTimestamp > maxAge otherwise.
     *
     * @param lastModifiedTimestamp The timestamp of a file/... to compare to the maxAge.
     * @param maxAge                The maximum age the lastModifiedTimestamp may be, before returning
     *                              true.
     * @return true if the timestamp is <= 0 or older than the maxAge.
     */
    protected boolean isUpdatedAgeOlderThan(long lastModifiedTimestamp, long maxAge) {
        if (lastModifiedTimestamp > 0) {
            final long currentTime = TimeUtils.utcNow();
            final long ageDifference = currentTime - lastModifiedTimestamp;
            return ageDifference > maxAge;
        } else {
            return true;
        }
    }

    protected void lockFile(File file) {
        final File directoryToPlaceLockIn = getParentIfDirectory(file);

        propertyFiles.addCsv(directoryToPlaceLockIn, "lock", LOCK_LOCKED_FILE.getKey(), file.getAbsolutePath());
        propertyFiles.set(directoryToPlaceLockIn, "lock", LOCK_LOCKED_SINCE.getKey(), TimeUtils.utcNow());

        this.lockedFiles.add(file.getAbsolutePath());

        propertyFiles.flushCachedAePropertyFiles();

        LOG.info("Locked file {}", file.getAbsolutePath());
    }

    protected void unlockFile(File file) {
        final File directoryToPlaceLockIn = getParentIfDirectory(file);

        final List lockedFiles = propertyFiles.getCsv(directoryToPlaceLockIn, "lock", LOCK_LOCKED_FILE.getKey());
        lockedFiles.remove(file.getAbsolutePath());

        propertyFiles.set(directoryToPlaceLockIn, "lock", LOCK_LOCKED_FILE.getKey(), lockedFiles);
        if (lockedFiles.size() == 0) {
            propertyFiles.remove(directoryToPlaceLockIn, "lock", LOCK_LOCKED_SINCE.getKey());
        } else {
            propertyFiles.set(directoryToPlaceLockIn, "lock", LOCK_LOCKED_SINCE.getKey(), TimeUtils.utcNow());
        }

        this.lockedFiles.remove(file.getAbsolutePath());

        propertyFiles.flushCachedAePropertyFiles();

        LOG.info("Unlocked file {}", file.getAbsolutePath());
    }

    protected void waitForFileUnlockIfLocked(File file, long timeout) {
        propertyFiles.flushCachedAePropertyFiles();

        if (isFileLocked(file, timeout)) {
            LOG.info("File is locked, waiting for unlock or timeout of [{}]: {}", formatMilliseconds(timeout), file.getAbsolutePath());
        } else {
            return;
        }

        do {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ignored) {
            }
        } while ((isFileLocked(file, timeout)));

        LOG.info("File has been unlocked: {}", file.getAbsolutePath());
    }

    private boolean isFileLocked(File file, long timeout) {
        final File directoryToPlaceLockIn = getParentIfDirectory(file);

        propertyFiles.discardCachedChangesForFileType("lock");

        final List lockedFiles = propertyFiles.getCsv(directoryToPlaceLockIn, "lock", LOCK_LOCKED_FILE.getKey());
        final boolean fileIsIncludedInLock = lockedFiles.contains(file.getAbsolutePath());

        if (!fileIsIncludedInLock) {
            return false;
        }

        final long lockedSince = propertyFiles.getLong(directoryToPlaceLockIn, "lock", LOCK_LOCKED_SINCE.getKey())
                .orElse(0L);

        return lockedSince == 0 || TimeUtils.utcNow() - lockedSince <= timeout;
    }

    protected enum InfoFileAttributes implements AbstractModelBase.Attribute {
        MIRROR_VERSION("mirror-version"),
        LAST_UPDATED("last-updated"),
        LAST_UPDATED_FORMATTED("last-updated-formatted"),
        LAST_CHECKED("last-checked"),
        LAST_CHECKED_FORMATTED("last-checked-formatted"),
        LAST_RESET("last-reset"),
        CERT_FR_PREFIX("cert-fr-"),
        CERT_SEI_PREFIX("cert-sei-"),
        CERT_EU_PREFIX("cert-eu-"),
        MSRC_PREFIX("msrc-"),
        CPE_DICTIONARY_PREFIX("cpe-dictionary-"),
        NVD_PREFIX("nvd-"),
        EPSS_PREFIX("epss-"),
        LOCK_LOCKED_FILE("locked-file"),
        LOCK_LOCKED_SINCE("locked-since"),
        DOWNLOAD_FAILED_FLAG("download-failed"),
        INDEX_FAILED_FLAG("index-failed");

        public final String key;

        InfoFileAttributes(String key) {
            this.key = key;
        }

        @Override
        public String getKey() {
            return this.key;
        }
    }

    private static File getParentIfDirectory(File file) {
        return file.isFile() ? file.getParentFile() : file;
    }

    protected String formatMilliseconds(long ms) {
        final long hours = TimeUnit.MILLISECONDS.toHours(ms);
        final long minutes = TimeUnit.MILLISECONDS.toMinutes(ms - TimeUnit.HOURS.toMillis(hours));
        final long seconds = TimeUnit.MILLISECONDS.toSeconds(ms - TimeUnit.HOURS.toMillis(hours) - TimeUnit.MINUTES.toMillis(minutes));

        return String.format("%02d:%02d:%02d", hours, minutes, seconds);
    }

    protected void logTitle(String status) {
        final String type = this instanceof Download ? "download" : "index";
        final String dirName = this.getClass().isAnnotationPresent(MirrorMetadata.class) ? this.getClass().getAnnotation(MirrorMetadata.class).directoryName() : "Unknown";
        final String mavenName = this.getClass().isAnnotationPresent(MirrorMetadata.class) ? this.getClass().getAnnotation(MirrorMetadata.class).mavenPropertyName() : "Unknown";
        final String title = status + type + " [" + dirName + " / " + mavenName + "]";
        LOG.info(InventoryEnricher.formatLogHeader(title));
        if (StringUtils.hasText(status)) {
            LOG.info("");
        }
    }

    @Override
    public String toString() {
        if (this.getClass().isAnnotationPresent(MirrorMetadata.class)) {
            final MirrorMetadata mirrorMetadata = this.getClass().getAnnotation(MirrorMetadata.class);
            return mirrorMetadata.directoryName() + " / " + mirrorMetadata.mavenPropertyName();
        } else {
            return super.toString();
        }
    }

    public MirrorMetadata getMirrorMetadata() {
        return this.getClass().getAnnotation(MirrorMetadata.class);
    }
}