org.dspace.statistics.util.StatisticsClient Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dspace-api Show documentation
DSpace core data model and service APIs.
There is a newer version: 8.0
/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 * http://www.dspace.org/license/
 */
package org.dspace.statistics.util;

import java.io.File;
import java.net.URL;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.logging.log4j.Logger;
import org.apache.tools.ant.taskdefs.Get;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.statistics.factory.StatisticsServiceFactory;
import org.dspace.statistics.service.SolrLoggerService;

/**
 * Class to load intermediate statistics files into Solr.
 *
 * @author Stuart Lewis
 */
public class StatisticsClient {
    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(StatisticsClient.class);

    /**
     * Default constructor
     */
    private StatisticsClient() { }

    /**
     * Print the help message
     *
     * @param options  The command line options the user gave
     * @param exitCode the system exit code to use
     */
    private static void printHelp(Options options, int exitCode) {
        // print the help message
        HelpFormatter myhelp = new HelpFormatter();
        myhelp.printHelp("StatisticsClient\n", options);
        System.exit(exitCode);
    }

    /**
     * Main method to run the statistics importer.
     *
     * @param args The command line arguments
     * @throws Exception If something goes wrong
     */
    public static void main(String[] args) throws Exception {
        CommandLineParser parser = new DefaultParser();

        Options options = new Options();

        options.addOption("u", "update-spider-files", false,
                          "Update Spider IP Files from internet into " +
                              DSpaceServicesFactory.getInstance().getConfigurationService()
                                                   .getProperty("dspace.dir") + "/config/spiders");

        options.addOption("m", "mark-spiders", false, "Update isBot Flag in Solr");
        options.addOption("f", "delete-spiders-by-flag", false, "Delete Spiders in Solr By isBot Flag");
        options.addOption("i", "delete-spiders-by-ip", false, "Delete Spiders in Solr By IP Address");
        options.addOption("o", "optimize", false, "Run maintenance on the SOLR index");
        options.addOption("b", "reindex-bitstreams", false, "Reindex the bitstreams to ensure we have the bundle name");
        options.addOption("e", "export", false,
                          "Export SOLR view statistics data to usage-statistics-intermediate-format");
        options.addOption("r", "remove-deleted-bitstreams", false,
                          "While indexing the bundle names remove the statistics about deleted bitstreams");
        options.addOption("s", "shard-solr-index", false,
                          "Split the data from the main Solr core into separate Solr cores per year");
        options.addOption("h", "help", false, "help");

        CommandLine line = parser.parse(options, args);

        // Did the user ask to see the help?
        if (line.hasOption('h')) {
            printHelp(options, 0);
        }

        SolrLoggerService solrLoggerService = StatisticsServiceFactory.getInstance().getSolrLoggerService();
        if (line.hasOption("u")) {
            StatisticsClient.updateSpiderFiles();
        } else if (line.hasOption('m')) {
            solrLoggerService.markRobotsByIP();
        } else if (line.hasOption('f')) {
            solrLoggerService.deleteRobotsByIsBotFlag();
        } else if (line.hasOption('i')) {
            solrLoggerService.deleteRobotsByIP();
        } else if (line.hasOption('o')) {
            solrLoggerService.optimizeSOLR();
        } else if (line.hasOption('b')) {
            solrLoggerService.reindexBitstreamHits(line.hasOption('r'));
        } else if (line.hasOption('e')) {
            solrLoggerService.exportHits();
        } else if (line.hasOption('s')) {
            solrLoggerService.shardSolrIndex();
        } else {
            printHelp(options, 0);
        }
    }

    /**
     * Method to update Spiders in config directory.
     */
    private static void updateSpiderFiles() {
        try {
            System.out.println("Downloading latest spider IP addresses:");

            // Get the list URLs to download from
            String[] urls = DSpaceServicesFactory.getInstance().getConfigurationService()
                                                 .getArrayProperty("solr-statistics.spiderips.urls");
            if ((urls == null) || (urls.length == 0)) {
                System.err.println(" - Missing setting from dspace.cfg: solr.spiderips.urls");
                System.exit(0);
            }

            // Get the location of spiders directory
            File spiders = new File(
                DSpaceServicesFactory.getInstance().getConfigurationService().getProperty("dspace.dir"),
                "config/spiders");

            if (!spiders.exists() && !spiders.mkdirs()) {
                log.error("Unable to create spiders directory");
            }

            for (String value : urls) {
                value = value.trim();
                System.out.println(" Downloading: " + value);

                URL url = new URL(value);

                Get get = new Get();
                get.setDest(new File(spiders, url.getHost() + url.getPath().replace("/", "-")));
                get.setSrc(url);
                get.setUseTimestamp(true);
                get.execute();

            }


        } catch (Exception e) {
            System.err.println(" - Error: " + e.getMessage());
            e.printStackTrace();
            System.exit(1);
        }
    }


}