Logo coherent WaveBurst  
Library Reference Guide
Logo
Toolbox.hh
Go to the documentation of this file.
1 /*
2 # Copyright (C) 2019 Gabriele Vedovato
3 #
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17 
18 /**********************************************************
19  * Package: Toolbox Class Library
20  * File name: Toolbox.hh
21  * Author: Gabriele Vedovato (vedovato@lnl.infn.it)
22  **********************************************************/
23 
24 
25 #ifndef TOOLBOX_HH
26 #define TOOLBOX_HH
27 
28 #include "TROOT.h"
29 #include "TMath.h"
30 #include "TSystem.h"
31 #include "TString.h"
32 #include "TTree.h"
33 #include "TFile.h"
34 #include "TRandom.h"
35 #include "TRandom3.h"
36 #include "TObjArray.h"
37 #include "TObjString.h"
38 #include "TSystemDirectory.h"
39 #include "TChain.h"
40 #include "TApplication.h"
41 #include "TH1I.h"
42 #include "TF1.h"
43 #include "TH1D.h"
44 #include "TStyle.h"
45 #include "TLegend.h"
46 #include "TCanvas.h"
47 #include "TGraphSmooth.h"
48 #include "TMacro.h"
49 
50 #include <string>
51 #include <iostream>
52 #include <fstream>
53 #include <stdlib.h>
54 #include <math.h>
55 #include <ctype.h>
56 #include <vector>
57 #include <string.h>
58 
59 #include "wavecomplex.hh"
60 #include "wavearray.hh"
61 #include "network.hh"
62 #include "Meyer.hh"
63 #include "netevent.hh"
64 
65 #include "FrameL.h"
66 
67 #include "History.hh"
68 #include "Toolfun.hh"
69 
70 #define LST_TREE_NAME "frl"
71 
72 enum CWB_CAT {
73  CWB_CAT0 = 0,
74  CWB_CAT1 = 1,
75  CWB_CAT2 = 2,
76  CWB_CAT3 = 3,
77  CWB_HVETO = 10,
78  CWB_PEM = 11,
79  CWB_EXC = 12,
80  CWB_USER = 13
81 };
82 
83 struct dqfile {
84  char ifo[32];
85  char file[1024];
87  double shift;
88  bool invert;
89  bool c4;
90 };
91 
92 struct frfile {
93  int start;
94  int stop;
95  int length;
96  vector<TString> file;
97 };
98 
99 struct slag {
100  int jobId; // job id : sequential progressive number
101  vector<int> slagId; // slag id vector : [0]=jobId - [1]=1/0 1=header slag - [2,..,nIFO+1] ifo slag
102  vector<int> segId; // seg id vector : [0,..,nIFO-1] ifo segment number
103 };
104 
105 struct mdcshift {
106  double startMDC;
107  double stopMDC;
108  double offset;
109 };
110 
111 struct ifoparms {
113  double latitude;
114  double longitude;
115  double elevation;
116  double AltX; // elevation of the x arm
117  double AzX; // azimut of the x arm (angle-deg from nord)
118  double AltY; // elevation of the y arm
119  double AzY; // azimut of the y arm (angle-deg from nord)
120 };
121 
122 static vector<double> DEFAULT_DOUBLE_VECTOR;
123 
124 using namespace std;
125 
126 namespace CWB {
127 
128 class Toolbox {
129 
130 public:
131 
132  /* ************************ */
133  /* * segment methods * */
134  /* ************************ */
135 
136  static vector<waveSegment>
137  readSegments(TString ifile);
138  static vector<waveSegment>
139  unionSegments(vector<waveSegment>& ilist);
140  static vector<waveSegment>
141  sortSegments(vector<waveSegment>& ilist);
142  static vector<waveSegment>
143  invertSegments(vector<waveSegment>& ilist);
144  static vector<waveSegment>
145  mergeSegLists(vector<waveSegment>& ilist1, vector<waveSegment>& ilist2);
146  static waveSegment
147  getMaxSeg(vector<waveSegment> list);
148  static double
149  getTimeSegList(vector<waveSegment> list);
150  static int
151  dumpSegList(vector<waveSegment> list, TString fName, bool c4=false);
152  static vector<waveSegment>
153  getSlagJobList(vector<waveSegment> ilist, int seglen=600);
154 
155  /* ************************ */
156  /* * dq methods * */
157  /* ************************ */
158 
159  static vector<waveSegment>
160  readSegList(dqfile DQF);
161  static vector<waveSegment>
162  readSegList(int nDQF, dqfile* DQF, CWB_CAT dqcat);
163  static void
164  setSlagShifts(slag SLAG, vector<TString> ifos, double segLen, int nDQF, dqfile* DQF);
165 
166  /* ************************ */
167  /* * job methods * */
168  /* ************************ */
169 
170  static vector<waveSegment>
171  getJobList(vector<waveSegment> ilist, double segLen=600.,
172  double segMLS=300., double segEdge=8.);
173  static vector<waveSegment>
174  getJobList(vector<waveSegment> cat1List, vector<waveSegment> cat2List,
175  double segLen=600., double segMLS=300., double segTHR=30., double segEdge=8.);
176  static void
177  dumpJobList(vector<waveSegment> ilist, TString fName, double segLen=600.,
178  double segMLS=300., double segEdge=8.);
179  static vector<waveSegment>
180  getSegList(int jobId, int nIFO, double segLen, double segMLS, double segEdge,
181  vector<waveSegment> dqList);
182  static double
183  getLiveTime(vector<waveSegment>& jobList, vector<waveSegment>& dqList);
184  static double
185  getLiveTime(vector<waveSegment>& jobList, vector<waveSegment>& dqList,
186  vector<double> shiftList);
187 
188  /* ************************ */
189  /* * slag methods * */
190  /* ************************ */
191 
192  static vector<slag>
193  getSlagList(size_t nIFO, size_t slagSize, int slagSegs,
194  int slagOff, size_t slagMin, size_t slagMax,
195  size_t* slagSite, char* slagFile=NULL);
196  static void
197  dumpSlagList(vector<slag> slagList, TString slagFile, bool slagOnly=false);
198  static slag
199  getSlag(vector<slag> slagList, int jobid);
200  static vector<slag>
201  getSlagList(vector<slag> slagList, vector<TString> ifos, double segLen,
202  double segMin, double segEdge, int nDQF, dqfile* iDQF, CWB_CAT dqcat);
203  static void
204  getSlagList(vector<slag>& slagList, int slagSize,
205  int slagRank, int nifo, vector<int>& id);
206  static vector<waveSegment>
207  getSegList(slag SLAG, vector<waveSegment> jobList, double segLen,
208  double segMLS, double segEdge, vector<waveSegment> dqList);
209 
210  /* ************************ */
211  /* * MDC methods * */
212  /* ************************ */
213 
214  static double
215  getMDCShift(mdcshift mshift, double time);
216  static int
217  shiftBurstMDCLog(std::vector<std::string>& mdcList,
218  vector<TString> ifos, double mdc_shift);
219  static TString
220  SetMDCLog(TString log, int pos, TString val);
221  static TString
222  SetMDCLog(TString log, int pos, double val);
223  static TString
224  GetMDCLog(TString log, int pos);
225  static int
226  GetMDCLogSize(TString log);
227 
228  /* ************************ */
229  /* * condor methods * */
230  /* ************************ */
231 
232  static int
233  createDagFile(vector<waveSegment> jobList, TString condor_dir, TString label,
234  int jobmin=0, int jobmax=0);
235  static int
236  createDagFile(vector<slag> slagList, TString condor_dir, TString label,
237  int jobmin=0, int jobmax=0);
238  static int
239  createDagFile(vector<int> jobList, TString condor_dir, TString label,
240  vector<TString> jobFiles, TString stage="CWB_STAGE_FULL", int jobmin=0, int jobmax=0);
241  static int
242  createDagFile(vector<waveSegment> jobList, TString condor_dir, TString label,
243  vector<TString> jobFiles, TString stage="CWB_STAGE_FULL", int jobmin=0, int jobmax=0);
244  static int
245  createDagFile(vector<slag> slagList, TString condor_dir, TString label,
246  vector<TString> jobFiles, TString stage="CWB_STAGE_FULL", int jobmin=0, int jobmax=0);
247  static int
248  createSubFile(TString label, TString condor_dir, TString out_dir, TString err_dir,
249  TString log_dir, TString ext="", TString condor_tag="");
250  static vector<int>
251  getCondorJobList(TString condor_dir, TString label);
252  static vector<float>
253  getJobBenchmark(TString ifName, int stageID, TString bench);
254  static vector<float>
255  getJobBenchmark(TString ifName, int stageID, int resID, int factorID, TString bench);
256  static TString
257  DAG2LSF(char* dagFile, char* data_label, char* nodedir, char* data_dir,
258  char* condor_dir, char* log_dir, char* output_dir, char* work_dir);
259 
260  /* ************************ */
261  /* * merge methods * */
262  /* ************************ */
263 
264  static vector<TString>
265  mergeCWBTrees(TString dir_name, bool simulation, TString odir,
266  TString label, bool brms=false, bool bvar=false, bool bpsm=false);
267  static void
268  mergeCWBTrees(vector<TString> fileList, bool simulation, TString odir,
269  TString label, bool brms=false, bool bvar=false, bool bpsm=false);
270  static vector<TString>
271  mergeCWBTrees(int nthreads, TString dir_name, bool simulation, TString odir,
272  TString label, bool brms=false, bool bvar=false, bool bpsm=false);
273  static void
274  mergeCWBTrees(int nthreads, vector<TString> fileList, bool simulation, TString odir,
275  TString label, bool brms=false, bool bvar=false, bool bpsm=false);
276  static void
277  mergeTrees(vector<TString> fileList, TString treeName, TString odir, TString ofName, bool bhistory);
278  static vector<TString>
279  readFileList(TString ifName);
280  static void
281  dumpFileList(vector<TString> fileList, TString ofName);
282  static vector<int>
283  getMergeJobList(TString merge_dir, TString label, int version);
284  static vector<int>
285  getMergeJobList(TString ifname, vector<TString>& jobFileList);
286  static vector<int>
287  getMergeJobList(TString ifname);
288 
289  /* ************************ */
290  /* * history methods * */
291  /* ************************ */
292 
293  static char*
294  readFile(TString ifName);
295  static char*
296  getEnvCWB();
297 
298  /* ************************ */
299  /* * spectra methods * */
300  /* ************************ */
301 
302  static void
303  makeSpectrum(wavearray<double>& psd, wavearray<double> x, double chuncklen=8,
304  double scratchlen=0, bool oneside=true);
305  static void
306  makeSpectrum(TString ofname, wavearray<double> x, double chuncklen=8,
307  double scratchlen=0, bool oneside=true);
308  static void
309  getSimNoise(wavearray<double>& u, TString fName, int seed, int run);
310  static void
311  convertSampleRate(wavearray<double> iw, wavearray<double> ow);
312  static void
313  convertSampleRate(wavearray<double> ix, wavearray<double> iy,
315  static wavearray<double>
316  GetDetectorPSD(TString fName, double fWidth=8192., double dFreq=1.);
317 
318  /* ************************ */
319  /* * system methods * */
320  /* ************************ */
321 
322  static vector<TString>
323  getFileListFromDir(TString dir_name, TString endString="",
324  TString beginString="", TString containString="",bool fast=false);
325  static std::map<int,TString>
326  getJobFileMapFromDir(TString dir_name, TString endString="",
327  TString beginString="", TString containString="",bool fast=false);
328  static bool
329  isFileExisting(TString fName);
330  static bool
331  checkFile(TString fName, bool question=false, TString message="");
332  static void
333  mkDir(TString dir, bool question=false, bool remove=true);
334  static bool
335  rmDir(TString dir, bool question=true);
336  static void
337  PrintProcInfo(TString str="");
338  static TString
339  getTemporaryFileName(TString label="temporary", TString extension="tmp");
340  static int
341  mksTemp(char *fTemplate) {return mkstemp(fTemplate);} // wrapper to system funcion (used in CINT)
342  static vector<TString>
343  sortStrings(vector<TString> ilist);
344 
345  /* ******************************** */
346  /* * post processing methods * */
347  /* ******************************** */
348 
349  static int
350  setVeto(TString ifName, TString idir, TString odir, vector<TString> ifos,
351  int nVDQF, dqfile* VDQF, int nDQF, dqfile* DQF, double segLen,
352  double segMLS, double segEdge);
353  static int
354  setCuts(TString ifName, TString idir, TString odir,
355  TString trname, TString cuts, TString olabel);
356  static int
357  setIFAR(TString ifName, TString idir, TString odir, TString trname,
358  TString sels, TString farFile, int irho, TString olabel, bool inclusive=true);
359  static int
360  setChunk(TString ifName, TString idir, TString odir, TString trname, int chunk);
361  static void
362  setUniqueEvents(TString ifwave, TString ofwave, int nIFO, int pp_irho);
363  static int
364  CombineCBC(vector<TString> ifwave, vector<TString> ifmdc, TString ofwave, TString ofmdc,
365  int nIFO, float fthr, TString msearch, TString infos="", int lag=0, int slag=0, float ifarthr=0.);
366  static double
367  getLiveTime(int nIFO, TChain& liv, wavearray<double>& Trun,
369  wavearray<double>& Tdlag, int lag_number=-1, int slag_number=-1, int dummy=0);
370  static double
371  getZeroLiveTime(int nIFO, TChain& liv, int dummy=0);
372  static void
373  doPoissonPlot(int nIFO, wavearray<double>* Wlag,
375  static bool
376  isLeafInTree(TTree* itree, TString leaf);
377  static int
378  setMultiplicity(TString ifName, TString idir, TString odir, int nIFO, double dTime);
379  static wavecomplex
380  getRate(double rho, double Tgap, int nIFO, TChain& wav, wavearray<int>& Wsel,
382  static int GetStepFunction(TString fName, vector<double>& x, vector<double>& y,
383  vector<double>& ex = DEFAULT_DOUBLE_VECTOR, vector<double>& ey = DEFAULT_DOUBLE_VECTOR);
384  static double GetStepFunction(TString option, TString fName, double V=0,
385  vector<double>& x = DEFAULT_DOUBLE_VECTOR, vector<double>& y = DEFAULT_DOUBLE_VECTOR,
386  vector<double>& ex = DEFAULT_DOUBLE_VECTOR, vector<double>& ey = DEFAULT_DOUBLE_VECTOR);
389 
390  /* ************************ */
391  /* * utility methods * */
392  /* ************************ */
393 
394  static void
395  resampleToPowerOfTwo(wavearray<double>& w);
396  static int
397  getJobId(TString file, TString fext="root");
398  static TString
399  getParameter(TString options, TString param="");
400  static TString
401  getFileName(FILE* fp);
402  static TString
403  getFileName(char* symlink);
404  static void
406  static TString
407  WriteFrameFile(wavearray<double> x, TString chName, TString frName,
409  static bool
410  question(TString question);
411  static TString
412  addCWBFlags(TMacro macro, TString ofname="");
413 
414  /* ************************************************** */
415  /* * Hilbert & Wigner-Ville Transforms methods * */
416  /* ************************************************** */
417 
418  static wavearray<double>
419  getHilbertTransform(wavearray<double> x);
420  static wavearray<double>
421  getHilbertEnvelope(wavearray<double> x);
422  static wavearray<double>
423  getHilbertIFrequency(wavearray<double> x);
424  static wavearray<double>
425  getWignerVilleTransform(wavearray<double> x);
426  static void
427  unWrapPhase(wavearray<float>& p);
428  static void
429  getSineFittingParams(double a, double b, double c, double rate,
430  double& amplitude, double& omega, double& phase);
431 
432  /* ************************************************** */
433  /* * Color blind set methods * */
434  /* ************************************************** */
435 
436  static Int_t
437  getTableau10BlindColor( Int_t index );
438  static Int_t
439  getTableau10BlindColor( TString name );
440  static void
441  getTableau10BlindColorPalette( const int nSteps, Int_t *palette );
442 
443 
444 private:
445 
446  static void
447  blackmanharris (double* window, int n);
448 
449  ClassDef(Toolbox,1)
450 };
451 
452 } // end namespace
453 
454 #endif
char ofile[1024]
double rho
double segMLS
Definition: test_config1.C:47
int slagSize
Definition: test_config1.C:65
double startMDC
Definition: Toolbox.hh:106
TString ofName
size_t * slagSite
Definition: test_config1.C:69
int slagOff
Definition: test_config1.C:68
int stop
Definition: Toolbox.hh:94
mdcshift mdc_shift
Definition: test_config1.C:93
TB createDagFile(rslagList, full_condor_dir, data_label, jobFiles, cwb_stage_name)
Definition: ced.hh:42
CWB run(runID)
wavearray< double > a(hp.size())
wavearray< double > Trun(500000)
cout<< "slagList size : "<< slagList.size()<< endl;cout<< endl<< "Start segments selection from dq cat1 list ..."<< endl<< endl;rslagList=TB.getSlagList(slagList, ifos, segLen, segMLS, segEdge, nDQF, DQF, CWB_CAT1);cout<< "Number of selected jobs after cat1 : "<< rslagList.size()<< endl;cout<< endl<< "Start segments selection from dq cat2 list ..."<< endl<< endl;rslagList=TB.getSlagList(rslagList, ifos, segLen, segTHR, segEdge, nDQF, DQF, CWB_CAT2);cout<< "Number of selected jobs after cat2 : "<< rslagList.size()<< endl;vector< TString > jobFiles
par [0] name
int n
Definition: cwb_net.C:28
double longitude
Definition: Toolbox.hh:114
dqfile VDQF[100]
bool invert
Definition: Toolbox.hh:88
TString("c")
int palette
Definition: DrawGnetwork2.C:17
char ifo[32]
Definition: Toolbox.hh:84
int start
Definition: Toolbox.hh:93
double stopMDC
Definition: Toolbox.hh:107
cout<< endl;cout<< "ts size = "<< ts.size()<< " ts rate = "<< ts.rate()<< endl;tf.Forward(ts, wdm);int levels=tf.getLevel();cout<< "tf size = "<< tf.size()<< endl;double dF=tf.resolution();double dT=1./(2 *dF);cout<< "rate(hz) : "<< RATE<< "\ layers : "<< nLAYERS<< "\ dF(hz) : "<< dF<< "\ dT(ms) : "<< dT *1000.<< endl;int itime=TIME_PIXEL_INDEX;int ifreq=FREQ_PIXEL_INDEX;int index=(levels+1) *itime+ifreq;double time=itime *dT;double freq=(ifreq >0) ? ifreq *dF :dF/4;cout<< endl;cout<< "PIXEL TIME = "<< time<< " sec "<< endl;cout<< "PIXEL FREQ = "<< freq<< " Hz "<< endl;cout<< endl;wavearray< double > x
double phase
bool c4
Definition: Toolbox.hh:89
CWB_CAT dqcat
char odir[1024]
double amplitude
wavearray< double > psd(33)
CWB::frame fr(FRLIST_NAME)
int slagMax
Definition: test_config1.C:67
static int mksTemp(char *fTemplate)
Definition: Toolbox.hh:341
vector< int > segId
Definition: Toolbox.hh:102
CWB_CAT
Definition: Toolbox.hh:72
STL namespace.
TB createSubFile(data_label, full_condor_dir, full_condor_out_dir, full_condor_err_dir, condor_log, extention, condor_tag)
static vector< double > DEFAULT_DOUBLE_VECTOR
Definition: Toolbox.hh:122
double segEdge
Definition: test_config1.C:49
int nVDQF
TB mkDir(cwb_skymap_dir, false, true)
wavearray< int > Wsel(ntrg)
TList * list
TB mergeTrees(fileList, TREE_NAME, OUTPUT_MERGE_DIR, OFILE_NAME, false)
int jobId
Definition: Toolbox.hh:100
double fWidth
Definition: DrawPSD.C:16
double Tgap
Definition: test_config1.C:23
nDQF
Definition: cwb_eced.C:109
double segTHR
Definition: test_config1.C:48
wavearray< double > w
Definition: Test1.C:27
#define nIFO
char val[20]
CWB_CAT cat
Definition: Toolbox.hh:86
char data_label[512]
Definition: test_config1.C:160
TString chName[NIFO_MAX]
char str[1024]
char file[1024]
Definition: Toolbox.hh:85
wavearray< double > Rlag
int jobmax
int length
Definition: Toolbox.hh:95
vector< int > jobList
ifwave[0]
vector< slag > slagList
TString label
Definition: MergeTrees.C:21
bool log
Definition: WaveMDC.C:41
dqfile DQF[12]
Definition: test_config1.C:171
TString frDir[NIFO_MAX]
wavearray< double > Tlag
segLen
Definition: cwb_eced.C:24
double shift
Definition: Toolbox.hh:87
char merge_dir[512]
Definition: test_config1.C:147
wavearray< double > Tdlag
char data_dir[512]
Definition: test_config1.C:152
double offset
Definition: Toolbox.hh:108
vector< TString > getFileListFromDir(TString dir_name, TString endString="", TString beginString="")
Definition: cwb_csh2sh.C:168
vector< waveSegment > cat1List
char log_dir[512]
Definition: test_config1.C:151
TFile * ifile
double elevation
Definition: Toolbox.hh:115
double AltY
Definition: Toolbox.hh:118
s s
Definition: cwb_net.C:155
float irho
char options[256]
TString frName[NIFO_MAX]
int nifo
double AltX
Definition: Toolbox.hh:116
bool inclusive
Definition: cwb_setifar.C:157
cout<< "Starting reading output directory ..."<< endl;vector< TString > fileList
wavearray< int > index
int jobmin
vector< int > slagId
Definition: Toolbox.hh:101
double AzY
Definition: Toolbox.hh:119
TString dir_name[NDIR]
Definition: cwb_mkdir.C:76
double latitude
Definition: Toolbox.hh:113
Definition: Toolbox.hh:99
cout<< "total cat1 livetime : "<< int(cat1_time)<< " sec "<< cat1_time/3600.<< " h "<< cat1_time/86400.<< " day"<< endl;cout<< endl;vector< waveSegment > cat2List
Definition: cwb_dump_job.C:40
double omega
Definition: testWDM_4.C:12
TTree * itree
wavearray< double > Wlag[NIFO_MAX+1]
slagFile
Definition: cwb_tune_slag.C:25
int jobId
char nodedir[1024]
Definition: test_config1.C:187
char condor_dir[512]
Definition: test_config1.C:148
double dFreq
Definition: DrawPSD.C:17
vector< TString > file
Definition: Toolbox.hh:96
char work_dir[512]
Definition: test_config1.C:143
int slagMin
Definition: test_config1.C:66
string version
Definition: cWB_conf.py:108
simulation
Definition: cwb_eced.C:26
char fName[256]
char output_dir[512]
Definition: test_config1.C:146
void getUniqueFileList(TString ifile, TString ofile)
Definition: FrDisplay.cc:1422
wavearray< double > y
Definition: Test10.C:31
TString frLabel[NIFO_MAX]
TString name
Definition: Toolbox.hh:112
i drho pp_irho
TB checkFile(gSystem->Getenv("CWB_ROOTLOGON_FILE"))
TString ifos[60]
double AzX
Definition: Toolbox.hh:117
TB dumpSegList(dq1List, dq1ListFile, false)
wavearray< double > Wslag[NIFO_MAX+1]
TString treeName
Definition: MergeTrees.C:15