weka.core.WekaPackageManager Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of weka-dev Show documentation
The Waikato Environment for Knowledge Analysis (WEKA), a machine learning workbench. This version represents the developer version, the "bleeding edge" of development, you could say. New functionality gets added to this version.
There is a newer version: 3.9.6
Show newest version
/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see .
 */

/*
 *    WekaPackageManager.java
 *    Copyright (C) 2009-2013 University of Waikato, Hamilton, New Zealand
 */

package weka.core;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.pentaho.packageManagement.DefaultPackageManager;
import org.pentaho.packageManagement.Dependency;
import org.pentaho.packageManagement.Package;
import org.pentaho.packageManagement.PackageConstraint;
import org.pentaho.packageManagement.PackageManager;

import weka.core.converters.ConverterUtils;
import weka.gui.GenericObjectEditor;
import weka.gui.GenericPropertiesCreator;
import weka.gui.beans.BeansProperties;
import weka.gui.beans.KnowledgeFlowApp;
import weka.gui.beans.PluginManager;
import weka.gui.explorer.ExplorerDefaults;

/**
 * Class providing package management and manipulation routines. Also provides a
 * command line interface for package management.
 * 
 * @author Mark Hall (mhall{[at]}pentaho{[dot]}com)
 * @version $Revision: 10626 $
 */
public class WekaPackageManager {

  /** The default folder name for Weka bits and bobs */
  private static String WEKAFILES_DIR_NAME = "wekafiles";

  /** Default path to where Weka's configuration and packages are stored */
  public static File WEKA_HOME = new File(System.getProperty("user.home")
    + File.separator + WEKAFILES_DIR_NAME);

  /** The default packages directory */
  public static File PACKAGES_DIR = new File(WEKA_HOME.toString()
    + File.separator + "packages");

  /** The default props dir name */
  private static String PROPERTIES_DIR_NAME = "props";

  /** The default properties directory */
  public static File PROPERTIES_DIR = new File(WEKA_HOME.toString()
    + File.separator + PROPERTIES_DIR_NAME);

  /** The underlying package manager */
  private static PackageManager PACKAGE_MANAGER = PackageManager.create();

  /** Current repository URL to use */
  private static URL REP_URL;

  /** Location of the repository cache */
  private static URL CACHE_URL;

  /** True if a cache build is required */
  private static boolean INITIAL_CACHE_BUILD_NEEDED = false;

  /** The name of the file that contains the list of packages in the repository */
  private static String PACKAGE_LIST_FILENAME = "packageListWithVersion.txt";

  /** Primary repository */
  private static String PRIMARY_REPOSITORY =
    "http://weka.sourceforge.net/packageMetaData";

  /** Backup mirror of the repository */
  private static String REP_MIRROR;

  /**
   * True if the user has specified a custom repository via a property in
   * PackageManager.props
   */
  private static boolean USER_SET_REPO = false;

  /** The package manager's property file */
  private static String PACKAGE_MANAGER_PROPS_FILE_NAME =
    "PackageManager.props";

  /** Operating offline? */
  public static boolean m_offline;

  /** Load packages? */
  private static boolean m_loadPackages = true;

  /** Established WEKA_HOME successfully? */
  protected static boolean m_wekaHomeEstablished;

  /** Packages loaded OK? */
  protected static boolean m_packagesLoaded;

  /** Package loading in progress? */
  public static boolean m_initialPackageLoadingInProcess = false;

  /* True if an initial cache build is needed and working offline */
  public static boolean m_noPackageMetaDataAvailable;

  /** The set of packages that the user has requested not to load */
  public static Set m_doNotLoadList;

  static {
    establishWekaHome();
  }

  /**
   * Establish WEKA_HOME if needed
   * 
   * @return true if WEKA_HOME was successfully established
   */
  protected static boolean establishWekaHome() {
    if (m_wekaHomeEstablished) {
      return true;
    }
    // look to see if WEKA_HOME has been defined as an environment
    // variable
    Environment env = Environment.getSystemWide();
    String wh = env.getVariableValue("WEKA_HOME");
    if (wh != null) {
      WEKA_HOME = new File(wh);
      PACKAGES_DIR = new File(wh + File.separator + "packages");
      PROPERTIES_DIR = new File(wh + File.separator + PROPERTIES_DIR_NAME);
    } else {
      env.addVariableSystemWide("WEKA_HOME", WEKA_HOME.toString());
    }

    boolean ok = true;
    if (!WEKA_HOME.exists()) {
      // create it for the user
      if (!WEKA_HOME.mkdir()) {
        System.err.println("Unable to create WEKA_HOME ("
          + WEKA_HOME.getAbsolutePath() + ")");
        ok = false;
      }
    }

    if (!PACKAGES_DIR.exists()) {
      // create the packages dir
      if (!PACKAGES_DIR.mkdir()) {
        System.err.println("Unable to create packages directory ("
          + PACKAGES_DIR.getAbsolutePath() + ")");
        ok = false;
      }
    }

    m_wekaHomeEstablished = ok;
    PACKAGE_MANAGER.setPackageHome(PACKAGES_DIR);

    m_doNotLoadList = getDoNotLoadList();
    try {
      // setup the backup mirror first
      // establishMirror();

      // user-supplied repository URL takes precedence over anything else
      String repURL =
        env.getVariableValue("weka.core.wekaPackageRepositoryURL");
      if (repURL == null || repURL.length() == 0) {
        // See if there is a URL named in
        // $WEKA_HOME/props/PackageRepository.props
        File repPropsFile =
          new File(PROPERTIES_DIR.toString() + File.separator
            + "PackageRepository.props");

        if (repPropsFile.exists()) {
          Properties repProps = new Properties();
          repProps.load(new FileInputStream(repPropsFile));
          repURL = repProps.getProperty("weka.core.wekaPackageRepositoryURL");
        }
      }

      if (repURL == null || repURL.length() == 0) {
        repURL = PRIMARY_REPOSITORY;
      } else {
        log(weka.core.logging.Logger.Level.INFO,
          "[WekaPackageManager] weka.core.WekaPackageRepositoryURL = " + repURL);
        // System.err.println("[WekaPackageManager] weka.core.WekaPackageRepositoryURL = "
        // + repURL);
        USER_SET_REPO = true;
      }

      REP_URL = new URL(repURL);
      PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL);
    } catch (MalformedURLException ex) {
      ex.printStackTrace();
    } catch (IOException ex) {
      ex.printStackTrace();
    }

    PACKAGE_MANAGER.setBaseSystemName("weka");
    PACKAGE_MANAGER.setBaseSystemVersion(weka.core.Version.VERSION);

    // Now check the cache and establish it if necessary
    File cacheDir =
      new File(WEKA_HOME.toString() + File.separator + "repCache");
    try {
      String tempCacheString = "file://" + cacheDir.toString();
      // sanitize URI and fix slashes (for Windows)
      tempCacheString = tempCacheString.replace(" ", "%20");
      tempCacheString = tempCacheString.replace('\\', '/');
      if (tempCacheString.startsWith("file://")
        && !tempCacheString.startsWith("file:///")) {
        tempCacheString = tempCacheString.substring(7);
        tempCacheString = "file:///" + tempCacheString;
      }
      URI tempURI = new URI(tempCacheString);
      // System.err.println(" -- " + tempURI.toString());
      CACHE_URL = tempURI.toURL();
    } catch (Exception e) {
      e.printStackTrace();
    }

    File packagesList =
      new File(cacheDir.getAbsolutePath() + File.separator
        + PACKAGE_LIST_FILENAME);
    if (!cacheDir.exists()) {
      if (!cacheDir.mkdir()) {
        System.err.println("Unable to create repository cache directory ("
          + cacheDir.getAbsolutePath() + ")");
        log(
          weka.core.logging.Logger.Level.WARNING,
          "Unable to create repository cache directory ("
            + cacheDir.getAbsolutePath() + ")");
        CACHE_URL = null;
      } else {
        // refreshCache();
        INITIAL_CACHE_BUILD_NEEDED = true;
      }
    }

    if (!packagesList.exists()) {
      INITIAL_CACHE_BUILD_NEEDED = true;
    }

    // Package manager general properties
    // Set via system props first
    String offline = env.getVariableValue("weka.packageManager.offline");
    if (offline != null) {
      m_offline = offline.equalsIgnoreCase("true");
    }
    String loadPackages =
      env.getVariableValue("weka.packageManager.loadPackages");
    if (loadPackages == null) {
      // try legacy
      loadPackages = env.getVariableValue("weka.core.loadPackages");
    }

    if (loadPackages != null) {
      m_loadPackages = loadPackages.equalsIgnoreCase("true");
    }

    // load any general package manager properties from props file
    File generalProps =
      new File(PROPERTIES_DIR.toString() + File.separator
        + PACKAGE_MANAGER_PROPS_FILE_NAME);
    if (generalProps.exists()) {
      Properties gProps = new Properties();
      try {
        gProps.load(new FileInputStream(generalProps));
        // this one takes precedence over the legacy one
        String repURL =
          gProps.getProperty("weka.core.wekaPackageRepositoryURL");
        if (repURL != null && repURL.length() > 0) {
          REP_URL = new URL(repURL);
          PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL);
        }

        offline = gProps.getProperty("weka.packageManager.offline");
        if (offline != null && offline.length() > 0) {
          m_offline = offline.equalsIgnoreCase("true");
        }

        loadPackages = gProps.getProperty("weka.packageManager.loadPackages");
        if (loadPackages == null) {
          // try legacy
          loadPackages = env.getVariableValue("weka.core.loadPackages");
        }
        if (loadPackages != null) {
          m_loadPackages = loadPackages.equalsIgnoreCase("true");
        }

        String pluginManagerDisableList =
          gProps.getProperty("weka.pluginManager.disable");
        if (pluginManagerDisableList != null
          && pluginManagerDisableList.length() > 0) {
          List disable = new ArrayList();
          String[] parts = pluginManagerDisableList.split(",");
          for (String s : parts) {
            disable.add(s.trim());
          }
          PluginManager.addToDisabledList(disable);
        }
      } catch (FileNotFoundException e) {
        e.printStackTrace();
      } catch (IOException e) {
        e.printStackTrace();
      }
    }

    if (INITIAL_CACHE_BUILD_NEEDED && m_offline) {
      m_noPackageMetaDataAvailable = true;
    }

    // Pass a valid http URL to the setProxyAuthentication()
    // method to ensure that a proxy or direct connection
    // is configured initially. This will prevent this method from
    // trying to configure the ProxySelector and proxy
    // via the file-based CACHE_URL somewhere down the track
    PACKAGE_MANAGER.setPackageRepositoryURL(REP_URL);
    PACKAGE_MANAGER.setProxyAuthentication(REP_URL);

    return ok;
  }

  /**
   * Establish the location of a mirror
   */
  protected static void establishMirror() {
    if (m_offline) {
      return;
    }

    try {
      String mirrorListURL =
        "http://www.cs.waikato.ac.nz/ml/weka/packageMetaDataMirror.txt";

      URLConnection conn = null;
      URL connURL = new URL(mirrorListURL);

      if (PACKAGE_MANAGER.setProxyAuthentication(connURL)) {
        conn = connURL.openConnection(PACKAGE_MANAGER.getProxy());
      } else {
        conn = connURL.openConnection();
      }

      conn.setConnectTimeout(10000); // timeout after 10 seconds
      conn.setReadTimeout(10000);

      BufferedReader bi =
        new BufferedReader(new InputStreamReader(conn.getInputStream()));

      REP_MIRROR = bi.readLine();

      bi.close();
      if (REP_MIRROR != null && REP_MIRROR.length() > 0) {
        // use the mirror if it is different from the primary repo
        // and the user hasn't specified an explicit repo via the
        // property
        if (!REP_MIRROR.equals(PRIMARY_REPOSITORY) && !USER_SET_REPO) {

          log(weka.core.logging.Logger.Level.INFO,
            "[WekaPackageManager] Package manager using repository mirror: "
              + REP_MIRROR);

          REP_URL = new URL(REP_MIRROR);
        }
      }
    } catch (Exception ex) {
      log(weka.core.logging.Logger.Level.WARNING,
        "[WekaPackageManager] The repository meta data mirror file seems "
          + "to be unavailable (" + ex.getMessage() + ")");
    }
  }

  /**
   * Write to the weka log
   * 
   * @param level logging level
   * @param message message to write
   */
  protected static void log(weka.core.logging.Logger.Level level, String message) {
    try {
      File logFile =
        new File(WEKA_HOME.toString() + File.separator + "weka.log");
      BufferedWriter writer = new BufferedWriter(new FileWriter(logFile, true));
      SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
      String linefeed = System.getProperty("line.separator");
      String m =
        format.format(new Date()) + " " + level + ": " + message + linefeed;
      writer.write(m);
      writer.flush();
      writer.close();
    } catch (Exception ex) {
    }
  }

  /**
   * Remove any ExplorerDefaults properties specified in the supplied package
   * 
   * @param installedPackageName the package specifying properties that should
   *          be removed from ExplorerDefaults
   */
  public static void removeExplorerProps(String installedPackageName) {
    try {
      Properties expProps = new Properties();
      String explorerProps =
        getPackageHome().getAbsolutePath() + File.separator
          + installedPackageName + File.separator + "Explorer.props";
      BufferedInputStream bi =
        new BufferedInputStream(new FileInputStream(explorerProps));
      expProps.load(bi);
      bi.close();
      bi = null;
      Set