ch.epfl.gsn.wrappers.SwissPegelWrapper Maven / Gradle / Ivy
The newest version!
/**
* Global Sensor Networks (GSN) Source Code
* Copyright (c) 2006-2016, Ecole Polytechnique Federale de Lausanne (EPFL)
*
* This file is part of GSN.
*
* GSN is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* GSN is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with GSN. If not, see not_a_number_constants = new ArrayList() ;
public boolean initialize() {
dateTimeFormat = new SimpleDateFormat( DateFormat );
svnDateTimeFormat = new SimpleDateFormat( SvnDateFormat );
sampling = getActiveAddressBean( ).getPredicateValueAsInt(SAMPLING, SAMPLING_DEFAULT);
directory = getActiveAddressBean().getPredicateValue(DIRECTORY);
datadirectory = getActiveAddressBean().getPredicateValue("data_directory");
svnurl = getActiveAddressBean().getPredicateValue(SVNURL);
if (datadirectory!=null && datadirectory.length()>0)
if (svnurl!=null && svnurl.length()>0){
if (directory==null||directory.length()==0){
logger.error("The wrapper failed, the "+DIRECTORY+" parameter is missing.");
return false;
}
file_handling = false;
svnlogin = getActiveAddressBean().getPredicateValue(SVNLOGIN);
svnpasswd = getActiveAddressBean().getPredicateValue(SVNPASSWD);
}
skip_lines = getActiveAddressBean().getPredicateValueAsInt(SKIP_LINES, 4);
String seperator_text = getActiveAddressBean().getPredicateValue(SEPERATOR);
String not_a_number_constant_val = getActiveAddressBean().getPredicateValue(NOT_A_NUMBER);
if (not_a_number_constant_val != null && not_a_number_constant_val.trim().length()>0) {
StringTokenizer st = new StringTokenizer(not_a_number_constant_val,",");
while (st.hasMoreTokens())
not_a_number_constants.add(st.nextToken().trim());
}
if (seperator_text.equals("tab")) seperator='\t';
else if (seperator_text.equals("space")) seperator=' ';
else seperator = seperator_text.charAt(0);
if (!readStatus()) return false;
logger.warn("wrapper correctly initialized");
return true;
}
private boolean readStatus(){
String filename;
if (file_handling) filename = datadirectory;
else filename = svnurl;
filename = filename.replace('/','_');
filename = filename.replace(':','_');
filename = filename.replace('\\','_');
statusFile = new File(directory+File.separator+filename+"_status.txt");
String contents = null;
if (statusFile.exists()){
try {
BufferedReader input = new BufferedReader(new FileReader(statusFile));
try {
String line = null; //not declared within while loop
while (( line = input.readLine()) != null){
contents = line;
}
}
finally {
input.close();
}
}
catch (IOException ex){
logger.error(ex.getMessage(), ex);
}
logger.warn("Content of the last line of the status file: "+contents);
if (contents!=null){
String[] list = contents.split(";");
logger.warn("number of split elements: "+list.length+" 0:"+list[0]+" 1:"+list[1]);
this.lastEnteredStreamelement = new Long(list[0]);
this.lastModified = new Long(list[1]);
} else {
this.lastEnteredStreamelement = new Long(0);
this.lastModified = new Long(0);
}
} else {
try {
statusFile.createNewFile();
} catch (IOException e) {
logger.error("the status file can not be created "+statusFile.getAbsolutePath());
return false;
}
}
return true;
}
private void writeStatus(){
try{
FileWriter fstream = new FileWriter(statusFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write(lastEnteredStreamelement+";"+lastModified);
out.close();
}catch (Exception e){//Catch exception if any
logger.error("Error: " + e.getMessage());
}
}
public StreamElement rowToSE(String[] data) {
Date date;
StreamElement se = null;
try {
date = dateTimeFormat.parse(data[1]+" "+data[0]);
se = new StreamElement(structure,removeTimestampFromRow(data),date.getTime());
} catch (ParseException e) {
logger.error("invalide date format! "+data[1]+" "+data[0]);
logger.error(e.getMessage(),e);
}finally {
return se;
}
}
public Double[] removeTimestampFromRow(String [] data) {
Double[] toReturn = new Double[structure.length];
next_val:for (int i=0;i list = getNewFileDataAvailable();
for (Long modified: list.keySet()){
File file = list.get(modified);
logger.warn("processing the received file list "+file.getAbsolutePath());
try {
String[] data = null;
reader = new CSVReader(new FileReader(file),seperator,'\"',skip_lines);
logger.warn("parse file "+file.getAbsolutePath());
while ((data =reader.readNext()) !=null) {
// if (data.length<(current_structure.length+1)) {
// logger.info("Possible empty line ignored.");
// continue;
// }
StreamElement streamElement = rowToSE(data);
if (streamElement.getTimeStamp()>this.lastEnteredStreamelement){
logger.warn("posting data");
postStreamElement(streamElement);
this.lastEnteredStreamelement = streamElement.getTimeStamp();
}
}
this.lastModified = modified.longValue();
writeStatus();
} catch (Exception e) {
logger.error("Error in reading/processing "+file);
logger.error(e.getMessage(),e);
} finally {
if (reader!=null)
try {
reader.close();
} catch (IOException e) {
}
}
}
} else {
logger.warn("start svn data processing");
TreeMap list = getNewSvnDataAvailable();
logger.warn("the list has been derived; there are elements: "+list.size());
for (Long modified:list.keySet()){
String name = list.get(modified);
logger.warn("processing the received file list "+name);
try {
String[] data = null;
Process p = Runtime.getRuntime().exec("svn cat "+name+" --username '"+svnlogin+"' --password '"+svnpasswd+"' ");
InputStream in = p.getInputStream();
BufferedReader d = new BufferedReader(new InputStreamReader(in));
reader = new CSVReader(d,seperator,'\"',skip_lines);
logger.warn("parse file "+name);
while ((data =reader.readNext()) !=null) {
// if (data.length<(current_structure.length+1)) {
// logger.info("Possible empty line ignored.");
// continue;
// }
StreamElement streamElement = rowToSE(data);
if (streamElement.getTimeStamp()>this.lastEnteredStreamelement){
logger.warn("posting data");
postStreamElement(streamElement);
this.lastEnteredStreamelement = streamElement.getTimeStamp();
}
}
this.lastModified = modified.longValue();
writeStatus();
} catch (Exception e) {
logger.error("Error in reading/processing "+name);
logger.error(e.getMessage(),e);
} finally {
if (reader!=null)
try {
reader.close();
} catch (IOException e) {
}
}
}
}
} catch (InterruptedException e){
logger.error(e.getMessage(), e);
}
}
}
private TreeMap getNewSvnDataAvailable(){
TreeMap nameList = new TreeMap();
try{
logger.warn("start getNewSvnDataAvailable()");
logger.warn("svnlogin:"+svnlogin+" svnpasswd:"+svnpasswd);
logger.warn("svnurl:"+svnurl);
String cmd = "svn info "+svnurl+"/ --username '"+svnlogin+"' --password '"+svnpasswd+"' -R --xml";
Process p = Runtime.getRuntime().exec(cmd);
logger.warn("process initialized");
InputStream in = p.getInputStream();
DOMParser parser = new DOMParser();
InputSource source = new InputSource(in);
parser.parse(source);
Document doc = parser.getDocument();
NodeList entries = doc.getElementsByTagName("entry");
logger.warn("entries: "+entries.getLength());
for(int i=0;i this.lastModified ){
nameList.put(new Long(date.getTime()),url);
logger.warn("add file: path name: "+name+" url: "+url+" date: "+dateStr);
}
}
}
} catch(IOException e){
logger.error("the svn can not be updated: "+e.getMessage());
} catch (SAXException e) {
logger.error("the svn created XML is not valid: "+e.getMessage());
} catch (DOMException e) {
logger.error("the xml provided by the svn resulted in a DOM excoption "+e.getMessage());
} catch (ParseException e) {
logger.error("the date format provided by the svn resulted in a parsing exception "+e.getMessage());
}
return nameList;
}
public DataField[] getOutputFormat() {
return structure;
}
public String getWrapperName() {
return "STS Piezometer Wrapper";
}
public void dispose() {
threadCounter--;
writeStatus();
}
/**
* scan the directory for new files and return the files in an ordered list according
* to their modification date for inserting the data; an empty collection is returned
* if there are no new files
* @return
*/
public TreeMap getNewFileDataAvailable(){
File dir = new File(datadirectory);
File[] list = dir.listFiles();
TreeMap map = new TreeMap();
long modified = this.lastModified;
for (int i=0;i10) && !(list[i].getName().startsWith("."))){
long l = list[i].lastModified();
if (l>this.lastModified) {
modified = l;
map.put(new Long(l), list[i]);
}
}
}
return map;
}
}