package sos.stacks.monitor;
import java.io.File;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Vector;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import sos.spooler.Job_impl;
import sos.spooler.Order;
import sos.spooler.Variable_set;
import sos.spooler.Web_service_request;
import sos.spooler.Web_service_response;
import sos.spooler.Xslt_stylesheet;
import sos.util.SOSClassUtil;
import sos.util.SOSDate;
import sos.util.SOSString;
import sos.xml.SOSXMLXPath;
/**
* @author andreas.pueschel@sos-berlin.com
* @author mueruevet.oeksuez@sos-berlin.com
*
*
* see job documentation in the package jobdoc for details
*/
public class JobSchedulerMonitorMessageJob extends Job_impl {
/** generate verbose output: 0=single line with minimal output, 1=single line with additional info, 2=multi line with configuration debug, 4=debug mode */
protected int verbosity = 0;
/** optional name for this job, defaults to scheduler job name */
protected String monitorJob = "JobSchedulerMonitorMessageJob";
/** optional message type: currently "status" and "reset" supported */
protected String messageType = "status";
/** persistent messages are repeatedly processed */
protected boolean isMessagePersistent = true;
/** Hilfsvariable */
protected Vector messageVariables = new Vector();
/** sos.util.SOSString Object */
private SOSString sosString = new SOSString();
/** optional nur der Job-Name soll überprüft werden*/
private String monitorJobname = "";
/** optional nur der Jobkette soll überprüft werden*/
private String monitorJobChainname = "";
/** optional, default = false, Loggen von Fehlern und Warnungen */
private boolean logMessages = false;
/** optional, default = true, beim = no versendet der Job keine Response (Web_service_response), */
private boolean NoWebServiceResponse = false;
/** Default Fehlermeldungen bzw. Warnungen, die ignoriert werden sollen.*/
private String includeSupervisorMsg = "[WARN](Supervisor;[ERROR](Supervisor";
/** Schalter, die Defaultmeldungen ausschaltet bzw. rausschreibt. */
private boolean includeSupervisor = false;
/** Parameter "exclude_messages": ergänzt (d.h. mergt) die Liste der in der Job-Implementierung ausgeschlossenen Messages
Messages werden als semikolon-separierte Liste auf Job- oder Auftragsparametern geliefert. Ist nur eine Message-ID übergeben,
dann kann das Semikolon fehlen; es ist kein Fehler wenn die Liste der Message-IDs mit Semikolon schließt.
*/
private String excludeMessages = null;
/** Patameter maximum_lifetime gibt die Lebensdauer einer Messages an. Die Messages kann in
*
* maximum_lifetime=hh24
* maximum_lifetime=hh24:mi
* maximum_lifetime=hh24:mi:ss
*
* angegeben werden.
*/
private String maxLifeTime = null;
/**
* Der Parameter bestimmt die maximale Anzahl, die Nachrichten an einen Network Monitor gesendet werden.
* Per Voreinstellung werden Nachrichten beliebig oft wiederholt gesendet.
*/
private int maximumReports = -1;
/**
* normalize message item
*/
protected String normalizeItem(String item) {
item = item.replaceAll("&", "&");
item = item.replaceAll("<", "<");
item = item.replaceAll(">", ">");
return item;
}
/**
* check scheduler log file for warnings and errors
*
*/
public Vector getLogMessages() {
Vector messages = new Vector();
HashMap message = new HashMap();
this.messageVariables = new Vector();
//Alle Meldungen, die ignoriert werden sollen
HashMap ignoreMSG = new HashMap();
try {
long logFilepointer = 0;
String logFilename = "";
String logLine = "";
File logFile = null;
File prevLogFile = null;
RandomAccessFile checkFile = null;
RandomAccessFile checkPrevFile = null;
ignoreMSG = getIgnoreMessages();
if (this.getMessageType().equalsIgnoreCase("reset")) {
spooler.set_var(getMonitorJob() + ".messages", "");
spooler_log.info("message stack has been reset");
message = new HashMap();
message.put("info", SOSDate.getCurrentTimeAsString() + ": warnings and errors were reset");
messages.add(message);
this.setMessageType("status");
}
try { // to retrieve persistent messages from scheduler variables
if (this.isMessagePersistent() ) {
String messageSet = spooler.var(this.getMonitorJob() + ".messages");
if (messageSet != null && messageSet.length() > 0) {
messageSet = messageSet.replaceAll(String.valueOf((char)254), "<").replaceAll(String.valueOf((char)255), ">");
SOSXMLXPath xpath = new SOSXMLXPath(new StringBuffer(messageSet));
NodeList nodelist = xpath.selectNodeList("//messages/*");
for (int i=0; i < nodelist.getLength(); i++) {
Node node = nodelist.item(i);
if (node == null) continue;
NodeList itemlist = xpath.selectNodeList(node, "*");
String curSeverity = "info";
String curContent = "";
for (int j=0; j<itemlist.getLength(); j++) {
Node item = itemlist.item(j);
if (item == null) continue;
if (item.getNodeName().equals("severity") && item.getFirstChild() != null) {
curSeverity = item.getFirstChild().getNodeValue();
} else if (item.getNodeName().equals("content") && item.getFirstChild() != null) {
curContent = item.getFirstChild().getNodeValue();
}
}
if( isValidMessages(curContent, ignoreMSG)) {
curContent = incrementReportCounter(curContent);
message = new HashMap();
spooler_log.debug9(".." + curSeverity +" = "+ curContent);
message.put(curSeverity, curContent);
messages.add(message);
}
}
}
}
} catch (Exception e) { // ignore all errors when processing persistent messages
spooler_log.info("persistent messages processed with errors: " + e.getMessage());
}
try { // to process possibly partially checked previous log files
String logFilepointerVariable = spooler.var(this.getMonitorJob() + ".filepointer");
String logFilenameVariable = spooler.var(this.getMonitorJob() + ".filename");
if (logFilepointerVariable != null && logFilepointerVariable.length() > 0)
logFilepointer = Long.parseLong(logFilepointerVariable);
if (logFilenameVariable != null && logFilenameVariable.length() > 0) logFilename = logFilenameVariable;
// on previous execution of this job a different log file may have been partially checked
if (logFilename != null && logFilename.length() > 0 && !logFilename.equals(new File(spooler.log().filename()).getCanonicalPath() )) {
prevLogFile = new File(logFilename);
if (!prevLogFile.exists()) {
spooler_log.info("log file from previous job execution not found: " + prevLogFile.getCanonicalPath() );
}
checkPrevFile = new RandomAccessFile(prevLogFile, "r");
if (logFilepointer > 0) {
checkPrevFile.seek(logFilepointer);
spooler_log.debug3("starting to check previous log file [" + prevLogFile.getCanonicalPath() + "] from position: " + logFilepointer);
}
//alte Fehlermeldung merken
while((logLine = checkPrevFile.readLine()) != null) {
if (logLine.indexOf("[WARN]") != -1) {
message = new HashMap();
spooler_log.debug9("..old WARN =" + logLine);
message.put("WARN", logLine);
messages.add(message);
} else if (logLine.indexOf("[ERROR]") != -1) {
message = new HashMap();
spooler_log.debug9("..old ERROR =" + logLine);
message.put("ERROR", logLine);
messages.add(message);
}
}
spooler_log.debug3("previous log file [" + prevLogFile.getCanonicalPath() + "] processed");
logFilepointer = 0;
logFilename = "";
}
} catch (Exception e) { // ignore all errors when processing previous log files (log files may have been gzipped or removed)
spooler_log.info("previous log file [" + ((prevLogFile != null) ? prevLogFile.getCanonicalPath() : "") + "] processed with errors: " + e.getMessage());
logFilepointer = 0;
logFilename = "";
} finally {
if (checkPrevFile != null ) try { checkPrevFile.close(); } catch (Exception ex) {} // gracefully ignore this error
}
try {
logFile = new File(spooler.log().filename());
if (!logFile.exists()) {
throw new Exception("log file not found: " + logFile.getCanonicalPath() );
}
checkFile = new RandomAccessFile(logFile, "r");
if (logFilepointer > 0) {
checkFile.seek(logFilepointer);
spooler_log.debug3("starting to check current log file [" + logFile.getCanonicalPath() + "] from position: " + logFilepointer);
}
ArrayList listOfJobs = new ArrayList();
if(sosString.parseToString(getMonitorJobname()).length()> 0) {
String[] split = getMonitorJobname().split(";");
for (int i = 0; i < split.length; i++)
listOfJobs.add(split[i]);
spooler_log.debug("..monitoring job: " + getMonitorJobname());
}
String currLogJobName = "";
String currOrderName = "";
HashMap currLogJobnames = new HashMap();
HashMap currOrdernames = new HashMap();
int countOfLines = 0;
while((logLine = checkFile.readLine()) != null) {
boolean loop = false;
String[] splitJobchainname = sosString.parseToString(this.getMonitorJobChainname()).split(";");
for (int i = 0; i < splitJobchainname.length; i++) {
//if(logLine.indexOf("SCHEDULER-842 Task is going to process Order " + this.getMonitorJobChainname() + ":") > -1 &&
if(logLine.indexOf("SCHEDULER-842 Task is going to process Order " + splitJobchainname[i] + ":") > -1 &&
logLine.indexOf("state=") > -1) {
String n = this.getCurrentJobname(logLine, true);
spooler_log.debug("..monitoring order job: " + n);
listOfJobs.add(n);
}
}
String jmn = "";
if(sosString.parseToString(this.getMonitorJobChainname()).length() > 0 && listOfJobs.size() == 0) {
continue;
}
for (int i = 0; i < listOfJobs.size(); i++) {
jmn = sosString.parseToString(listOfJobs.get(i));
if((logLine.indexOf(jmn.concat(":")) == -1) &&
(logLine.indexOf(jmn.concat(")")) == -1)) {
countOfLines++;
loop = true;
} else {
loop = false;
break;
}
}
if(loop) {
loop = false;
continue;
}
//order Jobs starting
if(logLine.indexOf("SCHEDULER-842 Task is going to process Order") > -1 &&
logLine.indexOf("state=") > -1 ) {
currOrderName = getCurrentOrdername(logLine);
currLogJobName = getCurrentJobname(logLine, true);
logLine = "[order=" + currOrderName + "]" + logLine;
if(sosString.parseToString(currOrderName).length() > 0 ){
currOrdernames.put(currOrderName, currLogJobName);
}
// currLogJobnames.put(currLogJobName, false);
currLogJobnames.put(currLogJobName, new Boolean(false));
spooler_log.debug6("..monitoring from [job=" + currLogJobName + "], [order=" + currOrderName + "]");
}
//Standalone Job starting
if(logLine.indexOf("SCHEDULER-930 Task ") > -1 && logLine.indexOf("started, cause:") > -1 &&
logLine.indexOf("order") == -1) {
if( (!currOrdernames.isEmpty() && !currOrdernames.containsValue(getCurrentJobname(logLine, true))) ||
( !currLogJobnames.containsKey(getCurrentJobname(logLine, false)))) {
currLogJobName = getCurrentJobname(logLine, false);
//currLogJobnames.put(currLogJobName, false);
currLogJobnames.put(currLogJobName, new Boolean(false));
spooler_log.debug6("..monitoring from [standalone job=" + currLogJobName + "]");
}
}
//job end
if( ((logLine.indexOf("(Task") > -1 && logLine.indexOf("state=closed") > -1) ||
(logLine.indexOf("SCHEDULER-843 Task has ended processing of Order") > -1 && (logLine.indexOf(sosString.parseToString(currOrdernames.get(currOrderName))) > -1)))
&& (currLogJobName.length() > 0)){
if (sosString.parseToString(currLogJobnames.get(currLogJobName)).length() > 0 &&
!sosString.parseToBoolean(currLogJobnames.get(currLogJobName)) ) {
if(sosString.parseToString(currOrderName).length() > 0) {
logLine = "[order=" + currOrderName + "]" + logLine;
}
Iterator msg = messages.iterator();
while (msg.hasNext()) {
HashMap h = (HashMap)msg.next();
if(sosString.parseToString(h.get("WARN")).indexOf("(Task " + currLogJobName) > -1 ||
sosString.parseToString(h.get("WARN")).indexOf("(Job " + currLogJobName) > - 1){
if(sosString.parseToString(h.get("WARN")).indexOf("[order=") > -1 ) {
if(sosString.parseToString(h.get("WARN")).startsWith("[order=" + currOrderName + "]")) {
spooler_log.debug9("..clear: " + h.get("WARN"));
h.clear();
}
} else {
spooler_log.debug9("..clear: " + h.get("WARN"));
h.clear();
}
}
if(sosString.parseToString(h.get("ERROR")).indexOf("(Task " + currLogJobName) > -1 ||
sosString.parseToString(h.get("ERROR")).indexOf("(Job " + currLogJobName) > - 1){
if(sosString.parseToString(h.get("WARN")).indexOf("[order=") > -1 ) {
if(sosString.parseToString(h.get("WARN")).startsWith("[order=" + currOrderName + "]")) {
spooler_log.debug6("..clear: " + h.get("WARN"));
h.clear();
}
} else {
spooler_log.debug6("..clear: " + h.get("WARN"));
h.clear();
}
}
}
}
int sPos = logLine.indexOf("(Task ") + 6;
int ePos = logLine.substring(sPos).indexOf(":");
spooler_log.debug6("..end monotoring [job=" + logLine.substring(sPos, sPos + ePos) + "]");
}
logLine = logLine + "[report_counter=1]";
if (logLine.matches( "^\\d{4}-\\d{1,2}-\\d{1,2} +\\d{1,2}:\\d{1,2}:\\d{1,2}(\\.\\d{1,3})? +\\[WARN\\] +.*" )
&& !isIgnoreMessages(logLine, ignoreMSG)) {
message = new HashMap();
currLogJobName = getCurrentJobname(logLine, false);
//currLogJobnames.put(currLogJobName, true);
currLogJobnames.put(currLogJobName, new Boolean(true));
if(sosString.parseToString(currOrderName).length() > 0) {
logLine = "[order=" + currOrderName + "]" + logLine;
}
spooler_log.debug9("..new WARN= " + logLine );
message.put("WARN", logLine);
messages.add(message);
} else if (logLine.matches( "^\\d{4}-\\d{1,2}-\\d{1,2} +\\d{1,2}:\\d{1,2}:\\d{1,2}(\\.\\d{1,3})? +\\[ERROR\\] +.*" )
&& !isIgnoreMessages(logLine, ignoreMSG)) {
message = new HashMap();
currLogJobName = getCurrentJobname(logLine, false);
currLogJobnames.put(currLogJobName, new Boolean(true));
if(sosString.parseToString(currOrderName).length() > 0) {
logLine = "[order=" + currOrderName + "]" + logLine;
}
spooler_log.debug9("..new ERROR= " + logLine);
message.put("ERROR", logLine);
messages.add(message);
}
}
if(countOfLines > 0) {
spooler_log.debug(".." + countOfLines + " line skip, cause no monitoring job");
countOfLines = 0;
}
Vector curMsg = new Vector();
Iterator logMsg = messages.iterator();
HashMap h = null;
while (logMsg.hasNext()) {
h = (HashMap)logMsg.next();
if(!h.isEmpty()) {
curMsg.add(h);
}
}
messages = curMsg;
logMsg = messages.iterator();
if(this.isLogMessages() || spooler_log.level() <= -3) {
spooler_log.info("The following warnings and errors are reported:");
while (logMsg.hasNext()) {
h = (HashMap)logMsg.next();
spooler_log.info(transformString(h));
}
}
spooler.set_var(this.getMonitorJob() + ".filename", logFile.getCanonicalPath());
spooler.set_var(this.getMonitorJob() + ".filepointer", Long.toString(checkFile.getFilePointer()));
spooler_log.debug3("current log file [" + logFile.getCanonicalPath() + "] processed to position: " + Long.toString(checkFile.getFilePointer()));
return messages;
} catch (Exception e) {
throw new Exception(e);
} finally {
if (checkFile != null ) try { checkFile.close(); } catch (Exception ex) {} // gracefully ignore this error
}
} catch (Exception e){
spooler_log.info("getLogMessages(): failed to check messages in log file [" + spooler.log().filename() + "]: " + e.getMessage());
return messages;
}
}
/**
* Extrahiert den Jobname von der Logzeile
*
* @param logLine
* @return String
* @throws Exception
*/
private String getCurrentJobname(String logLine, boolean withTaskId) throws Exception {
try {
int sPos = 0;
if(logLine.indexOf("(Task ") > -1)
sPos = logLine.indexOf("(Task ") + 6;
else if(logLine.indexOf("(Job ") > -1)
sPos = logLine.indexOf("(Job ") + 6;
else
sPos = logLine.indexOf("(") + 1;
int ePos = 0;
if(withTaskId || logLine.indexOf("(Job ") > -1) {
ePos = logLine.substring(sPos).indexOf(")");
} else if(logLine.indexOf("(Task ") > -1) {
ePos = logLine.substring(sPos).indexOf(":");
} else {
ePos = logLine.substring(sPos).indexOf(")");
}
String currLogJobName = "";
if(sPos == -1 || ePos == -1 || (sPos + ePos) <= -1)
currLogJobName = "";
else
currLogJobName = logLine.substring(sPos, sPos + ePos);
return currLogJobName ;
} catch (Exception e) {
spooler_log.info("error in " + SOSClassUtil.getMethodName() + ": " + e.getMessage());
return "";
}
}
/**
* Extrahiert den Auftragsname (=job_chain) von der Logzeile
*
* @param logLine
* @return String
* @throws Exception
*/
private String getCurrentOrdername(String logLine) throws Exception {
String currLogOrderName = "";
try {
if(logLine.indexOf("SCHEDULER-842 Task is going to process Order") > -1 &&
logLine.indexOf("state=start") > -1) {
int sPos = logLine.indexOf("Order ") + 6;
int ePos = logLine.substring(sPos |