com.metaeffekt.artifact.analysis.utils.DirectoryScanner Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-analysis Show documentation
The newest version!
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.metaeffekt.artifact.analysis.utils;

import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.json.JSONArray;
import org.json.JSONObject;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

/**
 * Recursively lists all files in a directory and returns them in a JSON object together with the time of scanning.
 */
public abstract class DirectoryScanner {

    /**
     * Scan a directory for any file type and any directory and return a list of all files with their metadata.
     *
     * @param baseDirectory The base directory to start scanning from.
     * @return A JSONObject that contains all files and some more metadata.
     * @throws IOException If the canonical path could not be generated for a file.
     */
    public static JSONObject scan(File baseDirectory) throws IOException {
        return scan(baseDirectory, new RegexFileFilter("^(.*?)"), DirectoryFileFilter.DIRECTORY, true);
    }

    /**
     * Scan a directory for any file type and any directory and return a list of all files with their metadata.
     *
     * @param baseDirectory      The base directory to start scanning from.
     * @param forceAbsolutePaths Whether to use absolute paths in the file listing or relative ones.
     * @return A JSONObject that contains all files and some more metadata.
     * @throws IOException If the canonical path could not be generated for a file.
     */
    public static JSONObject scan(File baseDirectory, boolean forceAbsolutePaths) throws IOException {
        return scan(baseDirectory, new RegexFileFilter("^(.*?)"), DirectoryFileFilter.DIRECTORY, forceAbsolutePaths);
    }

    /**
     * Scan a directory for specific file and directory types and return a list of all matching files with their metadata.
     *
     * @param baseDirectory       The base directory to start scanning from.
     * @param regexFileFilter     Only match specific files that match this filter.
     * @param directoryFileFilter Only match directories that match this filter.
     * @param forceAbsolutePaths  Whether to use absolute paths in the file listing or relative ones.
     * @return A JSONObject that contains all files and some more metadata.
     * @throws IOException If the canonical path could not be generated for a file.
     */
    public static JSONObject scan(File baseDirectory, RegexFileFilter regexFileFilter, IOFileFilter directoryFileFilter, boolean forceAbsolutePaths) throws IOException {
        JSONArray fileList = new JSONArray();

        for (File file : FileUtils.listFiles(baseDirectory, regexFileFilter, directoryFileFilter).stream().sorted().collect(Collectors.toList())) {
            JSONObject entry = new JSONObject();
            if (forceAbsolutePaths) entry.put(KEY_FILE_PATH, file.getCanonicalPath());
            else entry.put(KEY_FILE_PATH, file.getPath());
            entry.put(KEY_FILE_CHECKSUM, FileUtils.computeChecksum(file));
            entry.put(KEY_FILE_SIZE, file.length());
            fileList.put(entry);
        }

        JSONObject scanResult = new JSONObject();
        scanResult.put(KEY_TIMESTAMP, System.currentTimeMillis());
        scanResult.put(KEY_BASE_DIRECTORY, baseDirectory.getCanonicalPath());
        scanResult.put(KEY_AMOUNT_FILES, fileList.length());
        scanResult.put(KEY_FILES, fileList);
        return scanResult;
    }

    /**
     * Creates a list of all paths from a scanResult generated by the {@link #scan(File, RegexFileFilter, IOFileFilter, boolean)} function.
     *
     * @param scanResult The scan result to parse.
     * @return A list of all paths in the scan result.
     */
    public static List extractPathsFromResult(JSONObject scanResult) {
        if (scanResult == null) return new ArrayList<>();
        JSONArray fileEntries = scanResult.optJSONArray(KEY_FILES);
        if (fileEntries == null) return new ArrayList<>();
        List paths = new ArrayList<>();
        for (int i = 0; i < fileEntries.length(); i++) {
            JSONObject entry = fileEntries.optJSONObject(i);
            if (entry == null) continue;
            String path = entry.optString(KEY_FILE_PATH, null);
            if (path != null) paths.add(path);
        }
        return paths.stream().distinct().sorted().collect(Collectors.toList());
    }


    /**
     * Compares two scan results and detects:
     * Amount of files changed: {@link #KEY_AMOUNT_FILES}
     * Files removed: {@link #KEY_DIFFERENCE_FILE_REMOVED}
     * Files added: {@link #KEY_DIFFERENCE_FILE_ADDED}
     * Path changed but checksum still same: {@link #KEY_DIFFERENCE_PATH_DIFFERENT}
     * Checksum changed but path still same: {@link #KEY_DIFFERENCE_CHECKSUM_DIFFERENT}
     * File size changed but path and checksum still same: {@link #KEY_DIFFERENCE_SIZE_DIFFERENT}
     * 
     * Returns a {@link JSONArray} that contains all differences, showing what difference
     *
     * @param scanResult1 The scan result 1.
     * @param scanResult2 The scan result 2.
     * @return A {@link JSONArray} that contains all differences.
     */
    public static JSONArray compare(JSONObject scanResult1, JSONObject scanResult2) {
        if (scanResult1 == null || scanResult2 == null) return new JSONArray();
        JSONArray fileEntries1 = scanResult1.optJSONArray(KEY_FILES);
        JSONArray fileEntries2 = scanResult2.optJSONArray(KEY_FILES);
        if (fileEntries1 == null || fileEntries2 == null) return new JSONArray();

        JSONArray differences = new JSONArray();

        if (scanResult1.optInt(KEY_AMOUNT_FILES) != scanResult2.optInt(KEY_AMOUNT_FILES)) {
            JSONObject difference = new JSONObject();
            difference.put(KEY_DIFFERENCE_TYPE, KEY_DIFFERENCE_AMOUNT_DIFFERENT);
            difference.put(KEY_DIFFERENCE_AMOUNT_1, scanResult1.optString(KEY_AMOUNT_FILES));
            difference.put(KEY_DIFFERENCE_AMOUNT_2, scanResult2.optString(KEY_AMOUNT_FILES));
            differences.put(difference);
        }

        for (int i = 0; i < fileEntries1.length(); i++) {
            JSONObject entry1 = fileEntries1.optJSONObject(i);
            if (entry1 == null) continue;
            JSONObject entry2 = findEntry(fileEntries2, entry1);
            if (entry2 == null) {
                differences.put(makeDifferenceJson(entry1, KEY_DIFFERENCE_FILE_REMOVED));
                continue;
            }
            fileEntries2.remove(entry2.optInt("index", 0));

            String path1 = entry1.optString(KEY_FILE_PATH, null);
            if (path1 == null) continue;

            if (!entry1.optString(KEY_FILE_CHECKSUM).equals(entry2.optString(KEY_FILE_CHECKSUM))) {
                differences.put(makeDifferenceJson(entry1, entry2, KEY_DIFFERENCE_CHECKSUM_DIFFERENT));
            } else if (!entry1.optString(KEY_FILE_PATH).equals(entry2.optString(KEY_FILE_PATH))) {
                differences.put(makeDifferenceJson(entry1, entry2, KEY_DIFFERENCE_PATH_DIFFERENT));
            } else if (!entry1.optString(KEY_FILE_SIZE).equals(entry2.optString(KEY_FILE_SIZE))) {
                differences.put(makeDifferenceJson(entry1, entry2, KEY_DIFFERENCE_SIZE_DIFFERENT));
            }
        }

        for (int i = 0; i < fileEntries2.length(); i++) {
            JSONObject entry2 = fileEntries2.optJSONObject(i);
            if (entry2 == null) continue;
            differences.put(makeDifferenceJson(entry2, KEY_DIFFERENCE_FILE_ADDED));
        }

        return differences;
    }

    private static JSONObject makeDifferenceJson(JSONObject entry1, String type) {
        JSONObject difference = new JSONObject();
        difference.put(KEY_DIFFERENCE_TYPE, type);
        difference.put(KEY_DIFFERENCE_PATH_1, entry1.optString(KEY_FILE_PATH));
        difference.put(KEY_DIFFERENCE_CHECKSUM_1, entry1.optString(KEY_FILE_CHECKSUM));
        difference.put(KEY_DIFFERENCE_SIZE_1, entry1.optString(KEY_FILE_SIZE));
        return difference;
    }

    private static JSONObject makeDifferenceJson(JSONObject entry1, JSONObject entry2, String type) {
        JSONObject difference = new JSONObject();
        difference.put(KEY_DIFFERENCE_TYPE, type);
        difference.put(KEY_DIFFERENCE_PATH_1, entry1.optString(KEY_FILE_PATH));
        difference.put(KEY_DIFFERENCE_CHECKSUM_1, entry1.optString(KEY_FILE_CHECKSUM));
        difference.put(KEY_DIFFERENCE_SIZE_1, entry1.optString(KEY_FILE_SIZE));
        difference.put(KEY_DIFFERENCE_PATH_2, entry2.optString(KEY_FILE_PATH));
        difference.put(KEY_DIFFERENCE_CHECKSUM_2, entry2.optString(KEY_FILE_CHECKSUM));
        difference.put(KEY_DIFFERENCE_SIZE_2, entry2.optString(KEY_FILE_SIZE));
        return difference;
    }

    /**
     * Finds an entry in a JSONArray by first comparing the paths.
     * If no matching path was found, the checksums are compared.
     *
     * @param entries   The JSONArray that contains the entries to search the reference entry in.
     * @param reference The reference entry to pick the path and checksum from to search in the JSONArray.
     * @return The JSONObject entry from the JSONArray if a match was found or null if the reference does not exist in the array.
     */
    private static JSONObject findEntry(JSONArray entries, JSONObject reference) {
        for (int i = 0; i < entries.length(); i++) {
            JSONObject entry = entries.optJSONObject(i);
            if (entry == null) continue;
            if (entry.optString(KEY_FILE_PATH).equals(reference.optString(KEY_FILE_PATH))) {
                entry.put("index", i);
                return entry;
            }
        }
        for (int i = 0; i < entries.length(); i++) {
            JSONObject entry = entries.optJSONObject(i);
            if (entry == null) continue;
            if (entry.optString(KEY_FILE_CHECKSUM).equals(reference.optString(KEY_FILE_CHECKSUM))) {
                entry.put("index", i);
                return entry;
            }
        }
        return null;
    }

    public final static String KEY_TIMESTAMP = "timestamp";
    public final static String KEY_BASE_DIRECTORY = "baseDirectory";
    public final static String KEY_AMOUNT_FILES = "amount";
    public final static String KEY_FILES = "files";

    public final static String KEY_FILE_PATH = "path";
    public final static String KEY_FILE_CHECKSUM = "checksum";
    public final static String KEY_FILE_SIZE = "size";


    public final static String KEY_DIFFERENCE_TYPE = "type";
    public final static String KEY_DIFFERENCE_AMOUNT_DIFFERENT = "amountChanged";
    public final static String KEY_DIFFERENCE_FILE_REMOVED = "fileRemoved";
    public final static String KEY_DIFFERENCE_FILE_ADDED = "fileAdded";
    public final static String KEY_DIFFERENCE_CHECKSUM_DIFFERENT = "checksumChanged";
    public final static String KEY_DIFFERENCE_PATH_DIFFERENT = "pathChanged";
    public final static String KEY_DIFFERENCE_SIZE_DIFFERENT = "sizeChanged";

    public final static String KEY_DIFFERENCE_AMOUNT_1 = "amount1";
    public final static String KEY_DIFFERENCE_AMOUNT_2 = "amount2";
    public final static String KEY_DIFFERENCE_PATH_1 = "path1";
    public final static String KEY_DIFFERENCE_PATH_2 = "path2";
    public final static String KEY_DIFFERENCE_CHECKSUM_1 = "checksum1";
    public final static String KEY_DIFFERENCE_CHECKSUM_2 = "checksum2";
    public final static String KEY_DIFFERENCE_SIZE_1 = "size1";
    public final static String KEY_DIFFERENCE_SIZE_2 = "size2";
}