001 /* ============================================================
002 * JRobin : Pure java implementation of RRDTool's functionality
003 * ============================================================
004 *
005 * Project Info: http://www.jrobin.org
006 * Project Lead: Sasa Markovic (saxon@jrobin.org);
007 *
008 * (C) Copyright 2003-2005, by Sasa Markovic.
009 *
010 * Developers: Sasa Markovic (saxon@jrobin.org)
011 *
012 *
013 * This library is free software; you can redistribute it and/or modify it under the terms
014 * of the GNU Lesser General Public License as published by the Free Software Foundation;
015 * either version 2.1 of the License, or (at your option) any later version.
016 *
017 * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
018 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
019 * See the GNU Lesser General Public License for more details.
020 *
021 * You should have received a copy of the GNU Lesser General Public License along with this
022 * library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
023 * Boston, MA 02111-1307, USA.
024 */
025
026 package org.jrobin.data;
027
028 import org.jrobin.core.*;
029
030 import java.io.IOException;
031 import java.util.*;
032
033 /**
034 * Class which should be used for all calculations based on the data fetched from RRD files. This class
035 * supports ordinary DEF datasources (defined in RRD files), CDEF datasources (RPN expressions evaluation),
036 * SDEF (static datasources - extension of JRobin) and PDEF (plottables, see
037 * {@link Plottable Plottable} for more information.<p>
038 * <p/>
039 * Typical class usage:<p>
040 * <pre>
041 * final long t1 = ...
042 * final long t2 = ...
043 * DataProcessor dp = new DataProcessor(t1, t2);
044 * // DEF datasource
045 * dp.addDatasource("x", "demo.rrd", "some_source", "AVERAGE");
046 * // DEF datasource
047 * dp.addDatasource("y", "demo.rrd", "some_other_source", "AVERAGE");
048 * // CDEF datasource, z = (x + y) / 2
049 * dp.addDatasource("z", "x,y,+,2,/");
050 * // ACTION!
051 * dp.processData();
052 * // Dump calculated values
053 * System.out.println(dp.dump());
054 * </pre>
055 */
056 public class DataProcessor implements ConsolFuns {
057 /**
058 * Constant representing the default number of pixels on a JRobin graph (will be used if
059 * no other value is specified with {@link #setStep(long) setStep()} method.
060 */
061 public static final int DEFAULT_PIXEL_COUNT = 600;
062 private static final double DEFAULT_PERCENTILE = 95.0; // %
063
064 private int pixelCount = DEFAULT_PIXEL_COUNT;
065
066 /**
067 * Constant that defines the default {@link RrdDbPool} usage policy. Defaults to <code>false</code>
068 * (i.e. the pool will not be used to fetch data from RRD files)
069 */
070 public static final boolean DEFAULT_POOL_USAGE_POLICY = false;
071 private boolean poolUsed = DEFAULT_POOL_USAGE_POLICY;
072
073 private final long tStart;
074 private long tEnd, timestamps[];
075 private long lastRrdArchiveUpdateTime = 0;
076 // this will be adjusted later
077 private long step = 0;
078 // resolution to be used for RRD fetch operation
079 private long fetchRequestResolution = 1;
080
081 // the order is important, ordinary HashMap is unordered
082 private Map<String, Source> sources = new LinkedHashMap<String, Source>();
083
084 private Def[] defSources;
085
086 /**
087 * Creates new DataProcessor object for the given time span. Ending timestamp may be set to zero.
088 * In that case, the class will try to find the optimal ending timestamp based on the last update time of
089 * RRD files processed with the {@link #processData()} method.
090 *
091 * @param t1 Starting timestamp in seconds without milliseconds
092 * @param t2 Ending timestamp in seconds without milliseconds
093 * @throws RrdException Thrown if invalid timestamps are supplied
094 */
095 public DataProcessor(long t1, long t2) throws RrdException {
096 if ((t1 < t2 && t1 > 0 && t2 > 0) || (t1 > 0 && t2 == 0)) {
097 this.tStart = t1;
098 this.tEnd = t2;
099 }
100 else {
101 throw new RrdException("Invalid timestamps specified: " + t1 + ", " + t2);
102 }
103 }
104
105 /**
106 * Creates new DataProcessor object for the given time span. Ending date may be set to null.
107 * In that case, the class will try to find optimal ending date based on the last update time of
108 * RRD files processed with the {@link #processData()} method.
109 *
110 * @param d1 Starting date
111 * @param d2 Ending date
112 * @throws RrdException Thrown if invalid timestamps are supplied
113 */
114 public DataProcessor(Date d1, Date d2) throws RrdException {
115 this(Util.getTimestamp(d1), d2 != null ? Util.getTimestamp(d2) : 0);
116 }
117
118 /**
119 * Creates new DataProcessor object for the given time span. Ending date may be set to null.
120 * In that case, the class will try to find optimal ending date based on the last update time of
121 * RRD files processed with the {@link #processData()} method.
122 *
123 * @param gc1 Starting Calendar date
124 * @param gc2 Ending Calendar date
125 * @throws RrdException Thrown if invalid timestamps are supplied
126 */
127 public DataProcessor(Calendar gc1, Calendar gc2) throws RrdException {
128 this(Util.getTimestamp(gc1), gc2 != null ? Util.getTimestamp(gc2) : 0);
129 }
130
131 /**
132 * Returns boolean value representing {@link org.jrobin.core.RrdDbPool RrdDbPool} usage policy.
133 *
134 * @return true, if the pool will be used internally to fetch data from RRD files, false otherwise.
135 */
136 public boolean isPoolUsed() {
137 return poolUsed;
138 }
139
140 /**
141 * Sets the {@link org.jrobin.core.RrdDbPool RrdDbPool} usage policy.
142 *
143 * @param poolUsed true, if the pool should be used to fetch data from RRD files, false otherwise.
144 */
145 public void setPoolUsed(boolean poolUsed) {
146 this.poolUsed = poolUsed;
147 }
148
149 /**
150 * Sets the number of pixels (target graph width). This number is used only to calculate pixel coordinates
151 * for JRobin graphs (methods {@link #getValuesPerPixel(String)} and {@link #getTimestampsPerPixel()}),
152 * but has influence neither on datasource values calculated with the
153 * {@link #processData()} method nor on aggregated values returned from {@link #getAggregates(String)}
154 * and similar methods. In other words, aggregated values will not change once you decide to change
155 * the dimension of your graph.<p>
156 * <p/>
157 * The default number of pixels is defined by constant {@link #DEFAULT_PIXEL_COUNT}
158 * and can be changed with a {@link #setPixelCount(int)} method.
159 *
160 * @param pixelCount The number of pixels. If you process RRD data in order to display it on the graph,
161 * this should be the width of your graph.
162 */
163 public void setPixelCount(int pixelCount) {
164 this.pixelCount = pixelCount;
165 }
166
167 /**
168 * Returns the number of pixels (target graph width). See {@link #setPixelCount(int)} for more information.
169 *
170 * @return Target graph width
171 */
172 public int getPixelCount() {
173 return pixelCount;
174 }
175
176 /**
177 * Roughly corresponds to the --step option in RRDTool's graph/xport commands. Here is an explanation borrowed
178 * from RRDTool:<p>
179 * <p/>
180 * <i>"By default rrdgraph calculates the width of one pixel in the time
181 * domain and tries to get data at that resolution from the RRD. With
182 * this switch you can override this behavior. If you want rrdgraph to
183 * get data at 1 hour resolution from the RRD, then you can set the
184 * step to 3600 seconds. Note, that a step smaller than 1 pixel will
185 * be silently ignored."</i><p>
186 * <p/>
187 * I think this option is not that useful, but it's here just for compatibility.<p>
188 *
189 * @param step Time step at which data should be fetched from RRD files. If this method is not used,
190 * the step will be equal to the smallest RRD step of all processed RRD files. If no RRD file is processed,
191 * the step will be roughly equal to the with of one graph pixel (in seconds).
192 */
193 public void setStep(long step) {
194 this.step = step;
195 }
196
197 /**
198 * Returns the time step used for data processing. Initially, this method returns zero.
199 * Once {@link #processData()} is finished, the method will return the real value used for
200 * all internal computations. Roughly corresponds to the --step option in RRDTool's graph/xport commands.
201 *
202 * @return Step used for data processing.
203 */
204 public long getStep() {
205 return step;
206 }
207
208 /**
209 * Returns desired RRD archive step (reslution) in seconds to be used while fetching data
210 * from RRD files. In other words, this value will used as the last parameter of
211 * {@link RrdDb#createFetchRequest(String, long, long, long) RrdDb.createFetchRequest()} method
212 * when this method is called internally by this DataProcessor.
213 *
214 * @return Desired archive step (fetch resolution) in seconds.
215 */
216 public long getFetchRequestResolution() {
217 return fetchRequestResolution;
218 }
219
220 /**
221 * Sets desired RRD archive step in seconds to be used internally while fetching data
222 * from RRD files. In other words, this value will used as the last parameter of
223 * {@link RrdDb#createFetchRequest(String, long, long, long) RrdDb.createFetchRequest()} method
224 * when this method is called internally by this DataProcessor. If this method is never called, fetch
225 * request resolution defaults to 1 (smallest possible archive step will be chosen automatically).
226 *
227 * @param fetchRequestResolution Desired archive step (fetch resoltuion) in seconds.
228 */
229 public void setFetchRequestResolution(long fetchRequestResolution) {
230 this.fetchRequestResolution = fetchRequestResolution;
231 }
232
233 /**
234 * Returns ending timestamp. Basically, this value is equal to the ending timestamp
235 * specified in the constructor. However, if the ending timestamps was zero, it
236 * will be replaced with the real timestamp when the {@link #processData()} method returns. The real
237 * value will be calculated from the last update times of processed RRD files.
238 *
239 * @return Ending timestamp in seconds
240 */
241 public long getEndingTimestamp() {
242 return tEnd;
243 }
244
245 /**
246 * Returns consolidated timestamps created with the {@link #processData()} method.
247 *
248 * @return array of timestamps in seconds
249 * @throws RrdException thrown if timestamps are not calculated yet
250 */
251 public long[] getTimestamps() throws RrdException {
252 if (timestamps == null) {
253 throw new RrdException("Timestamps not calculated yet");
254 }
255 else {
256 return timestamps;
257 }
258 }
259
260 /**
261 * Returns calculated values for a single datasource. Corresponding timestamps can be obtained from
262 * the {@link #getTimestamps()} method.
263 *
264 * @param sourceName Datasource name
265 * @return an array of datasource values
266 * @throws RrdException Thrown if invalid datasource name is specified,
267 * or if datasource values are not yet calculated (method {@link #processData()}
268 * was not called)
269 */
270 public double[] getValues(String sourceName) throws RrdException {
271 Source source = getSource(sourceName);
272 double[] values = source.getValues();
273 if (values == null) {
274 throw new RrdException("Values not available for source [" + sourceName + "]");
275 }
276 return values;
277 }
278
279 /**
280 * Returns single aggregated value for a single datasource.
281 *
282 * @param sourceName Datasource name
283 * @param consolFun Consolidation function to be applied to fetched datasource values.
284 * Valid consolidation functions are MIN, MAX, LAST, FIRST, AVERAGE and TOTAL
285 * (these string constants are conveniently defined in the {@link ConsolFuns} class)
286 * @return MIN, MAX, LAST, FIRST, AVERAGE or TOTAL value calculated from the data
287 * for the given datasource name
288 * @throws RrdException Thrown if invalid datasource name is specified,
289 * or if datasource values are not yet calculated (method {@link #processData()}
290 * was not called)
291 */
292 public double getAggregate(String sourceName, String consolFun) throws RrdException {
293 Source source = getSource(sourceName);
294 return source.getAggregates(tStart, tEnd).getAggregate(consolFun);
295 }
296
297 /**
298 * Returns all (MIN, MAX, LAST, FIRST, AVERAGE and TOTAL) aggregated values for a single datasource.
299 *
300 * @param sourceName Datasource name
301 * @return Object containing all aggregated values
302 * @throws RrdException Thrown if invalid datasource name is specified,
303 * or if datasource values are not yet calculated (method {@link #processData()}
304 * was not called)
305 */
306 public Aggregates getAggregates(String sourceName) throws RrdException {
307 Source source = getSource(sourceName);
308 return source.getAggregates(tStart, tEnd);
309 }
310
311 /**
312 * This method is just an alias for {@link #getPercentile(String)} method.
313 * <p/>
314 * Used by ISPs which charge for bandwidth utilization on a "95th percentile" basis.<p>
315 * <p/>
316 * The 95th percentile is the highest source value left when the top 5% of a numerically sorted set
317 * of source data is discarded. It is used as a measure of the peak value used when one discounts
318 * a fair amount for transitory spikes. This makes it markedly different from the average.<p>
319 * <p/>
320 * Read more about this topic at
321 * <a href="http://www.red.net/support/resourcecentre/leasedline/percentile.php">Rednet</a> or
322 * <a href="http://www.bytemark.co.uk/support/tech/95thpercentile.html">Bytemark</a>.
323 *
324 * @param sourceName Datasource name
325 * @return 95th percentile of fetched source values
326 * @throws RrdException Thrown if invalid source name is supplied
327 */
328 public double get95Percentile(String sourceName) throws RrdException {
329 return getPercentile(sourceName);
330 }
331
332 /**
333 * Used by ISPs which charge for bandwidth utilization on a "95th percentile" basis.<p>
334 * <p/>
335 * The 95th percentile is the highest source value left when the top 5% of a numerically sorted set
336 * of source data is discarded. It is used as a measure of the peak value used when one discounts
337 * a fair amount for transitory spikes. This makes it markedly different from the average.<p>
338 * <p/>
339 * Read more about this topic at
340 * <a href="http://www.red.net/support/resourcecentre/leasedline/percentile.php">Rednet</a> or
341 * <a href="http://www.bytemark.co.uk/support/tech/95thpercentile.html">Bytemark</a>.
342 *
343 * @param sourceName Datasource name
344 * @return 95th percentile of fetched source values
345 * @throws RrdException Thrown if invalid source name is supplied
346 */
347 public double getPercentile(String sourceName) throws RrdException {
348 return getPercentile(sourceName, DEFAULT_PERCENTILE);
349 }
350
351 /**
352 * The same as {@link #getPercentile(String)} but with a possibility to define custom percentile boundary
353 * (different from 95).
354 *
355 * @param sourceName Datasource name.
356 * @param percentile Boundary percentile. Value of 95 (%) is suitable in most cases, but you are free
357 * to provide your own percentile boundary between zero and 100.
358 * @return Requested percentile of fetched source values
359 * @throws RrdException Thrown if invalid sourcename is supplied, or if the percentile value makes no sense.
360 */
361 public double getPercentile(String sourceName, double percentile) throws RrdException {
362 if (percentile <= 0.0 || percentile > 100.0) {
363 throw new RrdException("Invalid percentile [" + percentile + "], should be between 0 and 100");
364 }
365 Source source = getSource(sourceName);
366 return source.getPercentile(tStart, tEnd, percentile);
367 }
368
369 /**
370 * Returns array of datasource names defined in this DataProcessor.
371 *
372 * @return array of datasource names
373 */
374 public String[] getSourceNames() {
375 return sources.keySet().toArray(new String[0]);
376 }
377
378 /**
379 * Returns an array of all datasource values for all datasources. Each row in this two-dimensional
380 * array represents an array of calculated values for a single datasource. The order of rows is the same
381 * as the order in which datasources were added to this DataProcessor object.
382 *
383 * @return All datasource values for all datasources. The first index is the index of the datasource,
384 * the second index is the index of the datasource value. The number of datasource values is equal
385 * to the number of timestamps returned with {@link #getTimestamps()} method.
386 * @throws RrdException Thrown if invalid datasource name is specified,
387 * or if datasource values are not yet calculated (method {@link #processData()}
388 * was not called)
389 */
390 public double[][] getValues() throws RrdException {
391 String[] names = getSourceNames();
392 double[][] values = new double[names.length][];
393 for (int i = 0; i < names.length; i++) {
394 values[i] = getValues(names[i]);
395 }
396 return values;
397 }
398
399 private Source getSource(String sourceName) throws RrdException {
400 Source source = sources.get(sourceName);
401 if (source != null) {
402 return source;
403 }
404 throw new RrdException("Unknown source: " + sourceName);
405 }
406
407 /////////////////////////////////////////////////////////////////
408 // DATASOURCE DEFINITIONS
409 /////////////////////////////////////////////////////////////////
410
411 /**
412 * <p>Adds a custom, {@link org.jrobin.data.Plottable plottable} datasource (<b>PDEF</b>).
413 * The datapoints should be made available by a class extending
414 * {@link org.jrobin.data.Plottable Plottable} class.</p>
415 *
416 * @param name source name.
417 * @param plottable class that extends Plottable class and is suited for graphing.
418 */
419 public void addDatasource(String name, Plottable plottable) {
420 PDef pDef = new PDef(name, plottable);
421 sources.put(name, pDef);
422 }
423
424 /**
425 * <p>Adds complex source (<b>CDEF</b>).
426 * Complex sources are evaluated using the supplied <code>RPN</code> expression.</p>
427 * <p/>
428 * <p>Complex source <code>name</code> can be used:</p>
429 * <ul>
430 * <li>To specify sources for line, area and stack plots.</li>
431 * <li>To define other complex sources.</li>
432 * </ul>
433 * <p/>
434 * <p>JRobin supports the following RPN functions, operators and constants: +, -, *, /,
435 * %, SIN, COS, LOG, EXP, FLOOR, CEIL, ROUND, POW, ABS, SQRT, RANDOM, LT, LE, GT, GE, EQ,
436 * IF, MIN, MAX, LIMIT, DUP, EXC, POP, UN, UNKN, NOW, TIME, PI, E,
437 * AND, OR, XOR, PREV, PREV(sourceName), INF, NEGINF, STEP, YEAR, MONTH, DATE,
438 * HOUR, MINUTE, SECOND, WEEK, SIGN and RND.</p>
439 * <p/>
440 * <p>JRobin does not force you to specify at least one simple source name as RRDTool.</p>
441 * <p/>
442 * <p>For more details on RPN see RRDTool's
443 * <a href="http://people.ee.ethz.ch/~oetiker/webtools/rrdtool/manual/rrdgraph.html" target="man">
444 * rrdgraph man page</a>.</p>
445 *
446 * @param name source name.
447 * @param rpnExpression RPN expression containig comma (or space) delimited simple and complex
448 * source names, RPN constants, functions and operators.
449 */
450 public void addDatasource(String name, String rpnExpression) {
451 CDef cDef = new CDef(name, rpnExpression);
452 sources.put(name, cDef);
453 }
454
455 /**
456 * <p>Adds static source (<b>SDEF</b>). Static sources are the result of a consolidation function applied
457 * to *any* other source that has been defined previously.</p>
458 *
459 * @param name source name.
460 * @param defName Name of the datasource to calculate the value from.
461 * @param consolFun Consolidation function to use for value calculation
462 */
463 public void addDatasource(String name, String defName, String consolFun) {
464 SDef sDef = new SDef(name, defName, consolFun);
465 sources.put(name, sDef);
466 }
467
468 /**
469 * <p>Adds simple datasource (<b>DEF</b>). Simple source <code>name</code>
470 * can be used:</p>
471 * <ul>
472 * <li>To specify sources for line, area and stack plots.</li>
473 * <li>To define complex sources
474 * </ul>
475 *
476 * @param name source name.
477 * @param file Path to RRD file.
478 * @param dsName Datasource name defined in the RRD file.
479 * @param consolFunc Consolidation function that will be used to extract data from the RRD
480 * file ("AVERAGE", "MIN", "MAX" or "LAST" - these string constants are conveniently defined
481 * in the {@link org.jrobin.core.ConsolFuns ConsolFuns} class).
482 */
483 public void addDatasource(String name, String file, String dsName, String consolFunc) {
484 Def def = new Def(name, file, dsName, consolFunc);
485 sources.put(name, def);
486 }
487
488 /**
489 * <p>Adds simple source (<b>DEF</b>). Source <code>name</code> can be used:</p>
490 * <ul>
491 * <li>To specify sources for line, area and stack plots.</li>
492 * <li>To define complex sources
493 * </ul>
494 *
495 * @param name Source name.
496 * @param file Path to RRD file.
497 * @param dsName Data source name defined in the RRD file.
498 * @param consolFunc Consolidation function that will be used to extract data from the RRD
499 * file ("AVERAGE", "MIN", "MAX" or "LAST" - these string constants are conveniently defined
500 * in the {@link org.jrobin.core.ConsolFuns ConsolFuns} class).
501 * @param backend Name of the RrdBackendFactory that should be used for this RrdDb.
502 */
503 public void addDatasource(String name, String file, String dsName, String consolFunc, String backend) {
504 Def def = new Def(name, file, dsName, consolFunc, backend);
505 sources.put(name, def);
506 }
507
508 /**
509 * Adds DEF datasource with datasource values already available in the FetchData object. This method is
510 * used internally by JRobin and probably has no purpose outside of it.
511 *
512 * @param name Source name.
513 * @param fetchData Fetched data containing values for the given source name.
514 */
515 public void addDatasource(String name, FetchData fetchData) {
516 Def def = new Def(name, fetchData);
517 sources.put(name, def);
518 }
519
520 /////////////////////////////////////////////////////////////////
521 // CALCULATIONS
522 /////////////////////////////////////////////////////////////////
523
524 /**
525 * Method that should be called once all datasources are defined. Data will be fetched from
526 * RRD files, RPN expressions will be calculated, etc.
527 *
528 * @throws IOException Thrown in case of I/O error (while fetching data from RRD files)
529 * @throws RrdException Thrown in case of JRobin specific error
530 */
531 public void processData() throws IOException, RrdException {
532 extractDefs();
533 fetchRrdData();
534 fixZeroEndingTimestamp();
535 chooseOptimalStep();
536 createTimestamps();
537 assignTimestampsToSources();
538 normalizeRrdValues();
539 calculateNonRrdSources();
540 }
541
542 /**
543 * Method used to calculate datasource values which should be presented on the graph
544 * based on the desired graph width. Each value returned represents a single pixel on the graph.
545 * Corresponding timestamp can be found in the array returned from {@link #getTimestampsPerPixel()}
546 * method.
547 *
548 * @param sourceName Datasource name
549 * @param pixelCount Graph width
550 * @return Per-pixel datasource values
551 * @throws RrdException Thrown if datasource values are not yet calculated (method {@link #processData()}
552 * was not called)
553 */
554 public double[] getValuesPerPixel(String sourceName, int pixelCount) throws RrdException {
555 setPixelCount(pixelCount);
556 return getValuesPerPixel(sourceName);
557 }
558
559 /**
560 * Method used to calculate datasource values which should be presented on the graph
561 * based on the graph width set with a {@link #setPixelCount(int)} method call.
562 * Each value returned represents a single pixel on the graph. Corresponding timestamp can be
563 * found in the array returned from {@link #getTimestampsPerPixel()} method.
564 *
565 * @param sourceName Datasource name
566 * @return Per-pixel datasource values
567 * @throws RrdException Thrown if datasource values are not yet calculated (method {@link #processData()}
568 * was not called)
569 */
570 public double[] getValuesPerPixel(String sourceName) throws RrdException {
571 double[] values = getValues(sourceName);
572 double[] pixelValues = new double[pixelCount];
573 Arrays.fill(pixelValues, Double.NaN);
574 long span = tEnd - tStart;
575 // this is the ugliest nested loop I have ever made
576 for (int pix = 0, ref = 0; pix < pixelCount; pix++) {
577 double t = tStart + (double) (span * pix) / (double) (pixelCount - 1);
578 while (ref < timestamps.length) {
579 if (t <= timestamps[ref] - step) {
580 // too left, nothing to do, already NaN
581 break;
582 }
583 else if (t <= timestamps[ref]) {
584 // in brackets, get this value
585 pixelValues[pix] = values[ref];
586 break;
587 }
588 else {
589 // too right
590 ref++;
591 }
592 }
593 }
594 return pixelValues;
595 }
596
597 /**
598 * Calculates timestamps which correspond to individual pixels on the graph.
599 *
600 * @param pixelCount Graph width
601 * @return Array of timestamps
602 */
603 public long[] getTimestampsPerPixel(int pixelCount) {
604 setPixelCount(pixelCount);
605 return getTimestampsPerPixel();
606 }
607
608 /**
609 * Calculates timestamps which correspond to individual pixels on the graph
610 * based on the graph width set with a {@link #setPixelCount(int)} method call.
611 *
612 * @return Array of timestamps
613 */
614 public long[] getTimestampsPerPixel() {
615 long[] times = new long[pixelCount];
616 long span = tEnd - tStart;
617 for (int i = 0; i < pixelCount; i++) {
618 times[i] = Math.round(tStart + (double) (span * i) / (double) (pixelCount - 1));
619 }
620 return times;
621 }
622
623 /**
624 * Dumps timestamps and values of all datasources in a tabelar form. Very useful for debugging.
625 *
626 * @return Dumped object content.
627 * @throws RrdException Thrown if nothing is calculated so far (the method {@link #processData()}
628 * was not called).
629 */
630 public String dump() throws RrdException {
631 String[] names = getSourceNames();
632 double[][] values = getValues();
633 StringBuffer buffer = new StringBuffer();
634 buffer.append(format("timestamp", 12));
635 for (String name : names) {
636 buffer.append(format(name, 20));
637 }
638 buffer.append("\n");
639 for (int i = 0; i < timestamps.length; i++) {
640 buffer.append(format("" + timestamps[i], 12));
641 for (int j = 0; j < names.length; j++) {
642 buffer.append(format(Util.formatDouble(values[j][i]), 20));
643 }
644 buffer.append("\n");
645 }
646 return buffer.toString();
647 }
648
649 /**
650 * Returns time when last RRD archive was updated (all RRD files are considered).
651 *
652 * @return Last archive update time for all RRD files in this DataProcessor
653 */
654 public long getLastRrdArchiveUpdateTime() {
655 return lastRrdArchiveUpdateTime;
656 }
657
658 // PRIVATE METHODS
659
660 private void extractDefs() {
661 List<Def> defList = new ArrayList<Def>();
662 for (Source source : sources.values()) {
663 if (source instanceof Def) {
664 defList.add((Def) source);
665 }
666 }
667 defSources = defList.toArray(new Def[defList.size()]);
668 }
669
670 private void fetchRrdData() throws IOException, RrdException {
671 long tEndFixed = (tEnd == 0) ? Util.getTime() : tEnd;
672 for (int i = 0; i < defSources.length; i++) {
673 if (!defSources[i].isLoaded()) {
674 // not fetched yet
675 Set<String> dsNames = new HashSet<String>();
676 dsNames.add(defSources[i].getDsName());
677 // look for all other datasources with the same path and the same consolidation function
678 for (int j = i + 1; j < defSources.length; j++) {
679 if (defSources[i].isCompatibleWith(defSources[j])) {
680 dsNames.add(defSources[j].getDsName());
681 }
682 }
683 // now we have everything
684 RrdDb rrd = null;
685 try {
686 rrd = getRrd(defSources[i]);
687 lastRrdArchiveUpdateTime = Math.max(lastRrdArchiveUpdateTime, rrd.getLastArchiveUpdateTime());
688 FetchRequest req = rrd.createFetchRequest(defSources[i].getConsolFun(),
689 tStart, tEndFixed, fetchRequestResolution);
690 req.setFilter(dsNames);
691 FetchData data = req.fetchData();
692 defSources[i].setFetchData(data);
693 for (int j = i + 1; j < defSources.length; j++) {
694 if (defSources[i].isCompatibleWith(defSources[j])) {
695 defSources[j].setFetchData(data);
696 }
697 }
698 }
699 finally {
700 if (rrd != null) {
701 releaseRrd(rrd, defSources[i]);
702 }
703 }
704 }
705 }
706 }
707
708 private void fixZeroEndingTimestamp() throws RrdException {
709 if (tEnd == 0) {
710 if (defSources.length == 0) {
711 throw new RrdException("Could not adjust zero ending timestamp, no DEF source provided");
712 }
713 tEnd = defSources[0].getArchiveEndTime();
714 for (int i = 1; i < defSources.length; i++) {
715 tEnd = Math.min(tEnd, defSources[i].getArchiveEndTime());
716 }
717 if (tEnd <= tStart) {
718 throw new RrdException("Could not resolve zero ending timestamp.");
719 }
720 }
721 }
722
723 // Tricky and ugly. Should be redesigned some time in the future
724 private void chooseOptimalStep() {
725 long newStep = Long.MAX_VALUE;
726 for (Def defSource : defSources) {
727 long fetchStep = defSource.getFetchStep(), tryStep = fetchStep;
728 if (step > 0) {
729 tryStep = Math.min(newStep, (((step - 1) / fetchStep) + 1) * fetchStep);
730 }
731 newStep = Math.min(newStep, tryStep);
732 }
733 if (newStep != Long.MAX_VALUE) {
734 // step resolved from a RRD file
735 step = newStep;
736 }
737 else {
738 // choose step based on the number of pixels (useful for plottable datasources)
739 step = Math.max((tEnd - tStart) / pixelCount, 1);
740 }
741 }
742
743 private void createTimestamps() {
744 long t1 = Util.normalize(tStart, step);
745 long t2 = Util.normalize(tEnd, step);
746 if (t2 < tEnd) {
747 t2 += step;
748 }
749 int count = (int) (((t2 - t1) / step) + 1);
750 timestamps = new long[count];
751 for (int i = 0; i < count; i++) {
752 timestamps[i] = t1;
753 t1 += step;
754 }
755 }
756
757 private void assignTimestampsToSources() {
758 for (Source src : sources.values()) {
759 src.setTimestamps(timestamps);
760 }
761 }
762
763 private void normalizeRrdValues() throws RrdException {
764 Normalizer normalizer = new Normalizer(timestamps);
765 for (Def def : defSources) {
766 long[] rrdTimestamps = def.getRrdTimestamps();
767 double[] rrdValues = def.getRrdValues();
768 double[] values = normalizer.normalize(rrdTimestamps, rrdValues);
769 def.setValues(values);
770 }
771 }
772
773 private void calculateNonRrdSources() throws RrdException {
774 for (Source source : sources.values()) {
775 if (source instanceof SDef) {
776 calculateSDef((SDef) source);
777 }
778 else if (source instanceof CDef) {
779 calculateCDef((CDef) source);
780 }
781 else if (source instanceof PDef) {
782 calculatePDef((PDef) source);
783 }
784 }
785 }
786
787 private void calculatePDef(PDef pdef) {
788 pdef.calculateValues();
789 }
790
791 private void calculateCDef(CDef cDef) throws RrdException {
792 RpnCalculator calc = new RpnCalculator(cDef.getRpnExpression(), cDef.getName(), this);
793 cDef.setValues(calc.calculateValues());
794 }
795
796 private void calculateSDef(SDef sDef) throws RrdException {
797 String defName = sDef.getDefName();
798 String consolFun = sDef.getConsolFun();
799 Source source = getSource(defName);
800 double value = source.getAggregates(tStart, tEnd).getAggregate(consolFun);
801 sDef.setValue(value);
802 }
803
804 private RrdDb getRrd(Def def) throws IOException, RrdException {
805 String path = def.getPath(), backend = def.getBackend();
806 if (poolUsed && backend == null) {
807 return RrdDbPool.getInstance().requestRrdDb(path);
808 }
809 else if (backend != null) {
810 return new RrdDb(path, true, RrdBackendFactory.getFactory(backend));
811 }
812 else {
813 return new RrdDb(path, true);
814 }
815 }
816
817 private void releaseRrd(RrdDb rrd, Def def) throws IOException, RrdException {
818 String backend = def.getBackend();
819 if (poolUsed && backend == null) {
820 RrdDbPool.getInstance().release(rrd);
821 }
822 else {
823 rrd.close();
824 }
825 }
826
827 private static String format(String s, int length) {
828 StringBuffer b = new StringBuffer(s);
829 for (int i = 0; i < length - s.length(); i++) {
830 b.append(' ');
831 }
832 return b.toString();
833 }
834
835 /**
836 * Cute little demo. Uses demo.rrd file previously created by basic JRobin demo.
837 *
838 * @param args Not used
839 * @throws IOException
840 * @throws RrdException
841 */
842 public static void main(String[] args) throws IOException, RrdException {
843 // time span
844 long t1 = Util.getTimestamp(2003, 4, 1);
845 long t2 = Util.getTimestamp(2003, 5, 1);
846 System.out.println("t1 = " + t1);
847 System.out.println("t2 = " + t2);
848
849 // RRD file to use
850 String rrdPath = Util.getJRobinDemoPath("demo.rrd");
851
852 // constructor
853 DataProcessor dp = new DataProcessor(t1, t2);
854
855 // uncomment and run again
856 //dp.setFetchRequestResolution(86400);
857
858 // uncomment and run again
859 //dp.setStep(86500);
860
861 // datasource definitions
862 dp.addDatasource("X", rrdPath, "sun", "AVERAGE");
863 dp.addDatasource("Y", rrdPath, "shade", "AVERAGE");
864 dp.addDatasource("Z", "X,Y,+,2,/");
865 dp.addDatasource("DERIVE[Z]", "Z,PREV(Z),-,STEP,/");
866 dp.addDatasource("TREND[Z]", "DERIVE[Z],SIGN");
867 dp.addDatasource("AVG[Z]", "Z", "AVERAGE");
868 dp.addDatasource("DELTA", "Z,AVG[Z],-");
869
870 // action
871 long laptime = System.currentTimeMillis();
872 //dp.setStep(86400);
873 dp.processData();
874 System.out.println("Data processed in " + (System.currentTimeMillis() - laptime) + " milliseconds\n---");
875 System.out.println(dp.dump());
876
877 // aggregates
878 System.out.println("\nAggregates for X");
879 Aggregates agg = dp.getAggregates("X");
880 System.out.println(agg.dump());
881 System.out.println("\nAggregates for Y");
882 agg = dp.getAggregates("Y");
883 System.out.println(agg.dump());
884
885 // 95-percentile
886 System.out.println("\n95-percentile for X: " + Util.formatDouble(dp.get95Percentile("X")));
887 System.out.println("95-percentile for Y: " + Util.formatDouble(dp.get95Percentile("Y")));
888
889 // lastArchiveUpdateTime
890 System.out.println("\nLast archive update time was: " + dp.getLastRrdArchiveUpdateTime());
891 }
892 }
893